diff --git a/.github/workflows/pr-orchestrator.yml b/.github/workflows/pr-orchestrator.yml index 339999ce..2fb10aa2 100644 --- a/.github/workflows/pr-orchestrator.yml +++ b/.github/workflows/pr-orchestrator.yml @@ -643,6 +643,7 @@ jobs: echo "✅ Found coverage.xml" - name: Run quality gates run: | + set -o pipefail mkdir -p logs/quality-gates QG_LOG="logs/quality-gates/quality-gates_$(date -u +%Y%m%d_%H%M%S).log" { @@ -653,7 +654,7 @@ jobs: config = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) configured_fail_under = float(config["tool"]["coverage"]["report"]["fail_under"]) - print(max(configured_fail_under, 80.0)) + print(configured_fail_under) PY ) COVERAGE_PERCENT=$(python - <<'PY' diff --git a/CHANGELOG.md b/CHANGELOG.md index 35337e66..3f7fef65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,22 @@ All notable changes to this project will be documented in this file. --- +## [0.53.1] - 2026-07-17 + +### Fixed + +- Hardened native source readiness and corrected coverage-gate enforcement. + +## [0.53.0] - 2026-07-17 + +### Added + +- **Native source readiness**: reject incomplete Spec Kit artifacts and + policy-required invalid OpenSpec changes before requirement evidence is + normalized. + +--- + ## [0.52.3] - 2026-07-14 ### Fixed diff --git a/docs/reference/commands.generated.json b/docs/reference/commands.generated.json index 6889d413..e538aa81 100644 --- a/docs/reference/commands.generated.json +++ b/docs/reference/commands.generated.json @@ -2118,7 +2118,9 @@ "options": [ "--bundle", "--format", - "--from-file" + "--from-file", + "--from-openspec", + "--from-speckit" ], "owner_package": "nold-ai/specfact-requirements", "owner_repo": "nold-ai/specfact-cli-modules", diff --git a/docs/reference/commands.generated.md b/docs/reference/commands.generated.md index 61cce570..8189bf99 100644 --- a/docs/reference/commands.generated.md +++ b/docs/reference/commands.generated.md @@ -115,7 +115,7 @@ This file is generated from the current CLI command tree. Do not edit by hand. | `specfact project version set` | nold-ai/specfact-project | --bundle, --repo, --version; args: - | - | | | `specfact requirements` | nold-ai/specfact-requirements | --install-completion, --show-completion; args: - | coverage, import, list, validate | | | `specfact requirements coverage` | nold-ai/specfact-requirements | --bundle, --format; args: - | - | | -| `specfact requirements import` | nold-ai/specfact-requirements | --bundle, --format, --from-file; args: - | - | | +| `specfact requirements import` | nold-ai/specfact-requirements | --bundle, --format, --from-file, --from-openspec, --from-speckit; args: - | - | | | `specfact requirements list` | nold-ai/specfact-requirements | --bundle, --format, --show-coverage; args: - | - | | | `specfact requirements validate` | nold-ai/specfact-requirements | --bundle, --format, --profile; args: - | - | | | `specfact spec` | nold-ai/specfact-spec | --install-completion, --show-completion; args: - | backward-compat, generate-tests, mock, validate | | diff --git a/docs/reference/requirements-context-adapter.md b/docs/reference/requirements-context-adapter.md index 470a51a8..2ae72e0e 100644 --- a/docs/reference/requirements-context-adapter.md +++ b/docs/reference/requirements-context-adapter.md @@ -12,7 +12,7 @@ tracks: - src/specfact_cli/requirements/importers.py - tests/unit/requirements/test_context_adapter.py - tests/unit/requirements/test_upstream_evidence_imports.py -last_reviewed: 2026-07-13 +last_reviewed: 2026-07-17 exempt: false exempt_reason: "" --- @@ -64,6 +64,30 @@ version, and network-dependent parsing would make CI non-reproducible. To add an upstream format, add a pinned representative fixture, extend the core profile, and pass its compatibility test before release. +## Source Readiness + +Before emitting records, native imports reject incomplete Spec Kit sources with +an error diagnostic and no partial records. This includes recognized official +scaffold markers, unresolved `NEEDS CLARIFICATION` text, missing substantive +Functional Requirements, and a user story without a complete +Given/When/Then acceptance scenario. The initial compatibility fixture is +pinned to Spec Kit `v0.12.18`. + +OpenSpec imports remain portable by default. Native OpenSpec validation is +required only when the layered policy enables it: + +```yaml +validation: + openspec: + require_native_validation: true +``` + +The enterprise tier enables this policy by default; `strict` and +`enterprise_full_stack` resolve through that tier. When validation is required, +the importer runs `openspec validate --strict --json` with a bounded +timeout. A failed validation yields `source-invalid`; a missing executable +yields `upstream-validator-unavailable`. + ## Import Validation Gates For OpenSpec and Spec Kit records, `validate_requirement_context(...)` emits diff --git a/llms.txt b/llms.txt index 48981793..0ce95631 100644 --- a/llms.txt +++ b/llms.txt @@ -117,7 +117,7 @@ This file is generated from the current CLI command tree. Do not edit by hand. | `specfact project version set` | nold-ai/specfact-project | --bundle, --repo, --version; args: - | - | | | `specfact requirements` | nold-ai/specfact-requirements | --install-completion, --show-completion; args: - | coverage, import, list, validate | | | `specfact requirements coverage` | nold-ai/specfact-requirements | --bundle, --format; args: - | - | | -| `specfact requirements import` | nold-ai/specfact-requirements | --bundle, --format, --from-file; args: - | - | | +| `specfact requirements import` | nold-ai/specfact-requirements | --bundle, --format, --from-file, --from-openspec, --from-speckit; args: - | - | | | `specfact requirements list` | nold-ai/specfact-requirements | --bundle, --format, --show-coverage; args: - | - | | | `specfact requirements validate` | nold-ai/specfact-requirements | --bundle, --format, --profile; args: - | - | | | `specfact spec` | nold-ai/specfact-spec | --install-completion, --show-completion; args: - | backward-compat, generate-tests, mock, validate | | diff --git a/openspec/CHANGE_ORDER.md b/openspec/CHANGE_ORDER.md index 56d77284..83446cd2 100644 --- a/openspec/CHANGE_ORDER.md +++ b/openspec/CHANGE_ORDER.md @@ -8,7 +8,7 @@ active changes should be implemented. | Bucket | Count | Location | |---|---:|---| -| **Active** | 18 | [`openspec/changes/`](changes/) | +| **Active** | 19 | [`openspec/changes/`](changes/) | | **Parked** | 21 | [`openspec/parking-lot/`](parking-lot/) | | **Archived** | 115 | [`openspec/changes/archive/`](changes/archive/) | @@ -38,7 +38,7 @@ brownfield delivery. The active roadmap should make that thesis stronger: ## Active tracks -The 18 active changes group into three product tracks plus one reliability lane. +The 19 active changes group into three product tracks plus one reliability lane. Tracks can run in parallel; within a track, follow the order column. ### Track A - Validation Evidence Spine @@ -83,7 +83,8 @@ planning workflows. | 1 | `requirements-01-data-model` | [#238](https://github.com/nold-ai/specfact-cli/issues/238) | Normalized requirements-input records for evidence | arch-07 | | 2 | `requirements-02-module-commands` | [#239](https://github.com/nold-ai/specfact-cli/issues/239) | Import and normalize existing requirement context | requirements-01 | | 3 | `openspec-01-intent-trace` | [#350](https://github.com/nold-ai/specfact-cli/issues/350) | Import-first OpenSpec and Spec Kit requirement evidence with pass/fail gates (rescoped 2026-07-13) | requirements-01/02 | -| 4 | `architecture-01-solution-layer` | [#240](https://github.com/nold-ai/specfact-cli/issues/240) | Architecture-boundary records and drift validation | requirements input contracts | +| 4 | `requirements-04-upstream-source-readiness` | [#648](https://github.com/nold-ai/specfact-cli/issues/648) | Reject incomplete or policy-invalid native OpenSpec and Spec Kit sources before requirement normalization | openspec-01; paired modules #346 | +| 5 | `architecture-01-solution-layer` | [#240](https://github.com/nold-ai/specfact-cli/issues/240) | Architecture-boundary records and drift validation | requirements input contracts | | Parked | `requirements-03-backlog-sync` | [#244](https://github.com/nold-ai/specfact-cli/issues/244) | Read-first drift evidence; no write-back critical path. Deprioritized 2026-07-13 behind openspec-01 | requirements-02; modules `sync-01` | | Gated | `architecture-02-well-architected-review` | [#524](https://github.com/nold-ai/specfact-cli/issues/524) | Architecture-boundary review findings | architecture-01 shipped plus one usage cycle | | Gated | `telemetry-01-opentelemetry-default-on` | [#518](https://github.com/nold-ai/specfact-cli/issues/518) | Opt-in validation outcome telemetry only | governance-01 evidence fields | @@ -110,6 +111,7 @@ Update each proposal first, then run strict OpenSpec validation. | `requirements-01-data-model` | Reduce to optional normalized requirements-input records for validation evidence. | | `requirements-02-module-commands` | Drop requirement authoring as a flagship path. Keep import, normalization, validation, and coverage inspection. | | `requirements-03-backlog-sync` | Make drift detection/read-first import the product value. Keep write-back preview out of the critical path. | +| `requirements-04-upstream-source-readiness` | Keep source readiness core-owned and atomic; do not create an upstream authoring schema or require the OpenSpec CLI outside explicit or strict/enterprise policy. | | `architecture-01-solution-layer` | Reduce to architecture-boundary validation and drift evidence. Do not generate architecture. | | `openspec-01-intent-trace` | Done 2026-07-13: rescoped to import-first adapter consuming native OpenSpec and Spec Kit artifacts with deterministic pass/fail gates. | | `dogfooding-01-full-chain-e2e-proof` | Rewrite proof around real PR review, JSON evidence, AI-bloat findings, remediation packets, rerun proof. | @@ -161,6 +163,8 @@ Update each proposal first, then run strict OpenSpec validation. - `openspec-01` as import-first OpenSpec and Spec Kit requirement evidence adapter with deterministic gates; pulled forward to Track C order 3 on 2026-07-13 (rescoped, no longer positioned as optional-only). +- `requirements-04` as the core-owned source-readiness follow-up to + `openspec-01`; it blocks the paired modules command/persistence patch. - `architecture-02`, `telemetry-01`, and `ai-integration-02` only after pull from the validation loop exists. diff --git a/openspec/changes/requirements-04-upstream-source-readiness/.openspec.yaml b/openspec/changes/requirements-04-upstream-source-readiness/.openspec.yaml new file mode 100644 index 00000000..ca6baff6 --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/.openspec.yaml @@ -0,0 +1,5 @@ +schema: spec-driven +created: 2026-07-14 +goal: Provide a core-owned, deterministic source-readiness contract that + protects evidence integrity without creating an upstream authoring schema or + hard runtime dependency. diff --git a/openspec/changes/requirements-04-upstream-source-readiness/CHANGE_VALIDATION.md b/openspec/changes/requirements-04-upstream-source-readiness/CHANGE_VALIDATION.md new file mode 100644 index 00000000..551169e9 --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/CHANGE_VALIDATION.md @@ -0,0 +1,59 @@ +# Change Validation Report: requirements-04-upstream-source-readiness + +**Validation Date**: 2026-07-17 Europe/Berlin +**Change Proposal**: [proposal.md](./proposal.md) +**Validation Method**: Interface dry run in `/private/tmp/specfact-validation-requirements-04`, dependency search, strict OpenSpec validation, focused tests, and repository quality-gate verification. + +## Executive Summary + +- Breaking Changes: 0 detected +- Dependent Files: 3 affected +- Impact Level: Medium +- Validation Result: Pass +- User Decision: Resolved the readiness policy as `validation.openspec.require_native_validation`; the enterprise tier defaults it to `true`, and the compatibility fixture is pinned to Spec Kit `v0.12.18`. + +## Breaking Changes Detected + +None. `import_openspec_change` adds optional keyword-only `profile` and +`project_root` parameters, preserving its existing one-argument call contract. + +## Dependencies Affected + +### Critical Updates Required + +- `src/specfact_cli/requirements/importers.py`: add atomic readiness gates before normalization. +- `src/specfact_cli/requirements/context.py`: resolve the layered native-validation policy. + +### Recommended Updates + +- `tests/unit/requirements/test_upstream_evidence_imports.py`: fixture-backed source-readiness and policy tests. +- `docs/reference/requirements-context-adapter.md`: document diagnostics and layered policy behavior. + +## Impact Assessment + +- **Code Impact**: Read-only source import behavior becomes fail-closed for incomplete inputs; accepted source IDs, revisions, and scenarios remain unchanged. +- **Test Impact**: Native importer and profile-resolution regression coverage. +- **Quality-Gate Impact**: The workflow uses the configured line-coverage threshold and propagates quality-gate failures through its log pipeline. +- **Documentation Impact**: Requirements adapter reference documents readiness diagnostics and policy configuration. +- **Release Impact**: Minor feature; version/release work remains a delivery task after this PR is accepted. + +## Format Validation + +- **proposal.md Format**: Pass under repository strict OpenSpec validation. +- **tasks.md Format**: Pass; tests precede production edits and evidence is recorded in `TDD_EVIDENCE.md`. +- **specs Format**: Pass; scenarios cover incomplete sources, required validator failure, unavailable validators, and portable imports. +- **Config.yaml Compliance**: Pass. + +## OpenSpec Validation + +- **Status**: Pass +- **Command**: `openspec validate requirements-04-upstream-source-readiness --strict --json` +- **Issues Found/Fixed**: 0 + +## Validation Artifacts + +- Temporary interface workspace: `/private/tmp/specfact-validation-requirements-04` +- Failing-before and passing-after commands: [TDD_EVIDENCE.md](./TDD_EVIDENCE.md) +- The sibling internal repository is available, but it has no + `wiki/sources/requirements-04-upstream-source-readiness.md` mirror to update; + create that tracking page before release coordination if it becomes required. diff --git a/openspec/changes/requirements-04-upstream-source-readiness/README.md b/openspec/changes/requirements-04-upstream-source-readiness/README.md new file mode 100644 index 00000000..789bba4c --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/README.md @@ -0,0 +1,3 @@ +# requirements-04-upstream-source-readiness + +Reject incomplete or policy-invalid native OpenSpec and Spec Kit sources before normalizing requirements evidence. diff --git a/openspec/changes/requirements-04-upstream-source-readiness/TDD_EVIDENCE.md b/openspec/changes/requirements-04-upstream-source-readiness/TDD_EVIDENCE.md new file mode 100644 index 00000000..60b76857 --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/TDD_EVIDENCE.md @@ -0,0 +1,42 @@ +# TDD Evidence: requirements-04-upstream-source-readiness + +## Failing-before + +- **Date:** 2026-07-17 (Europe/Berlin) +- **Command:** `hatch run test -- tests/unit/requirements/test_upstream_evidence_imports.py` +- **Result:** 6 failed, 22 passed. +- **Expected failures:** the pristine v0.12.18 Spec Kit scaffold emitted six + placeholder records; incomplete Spec Kit sources returned the prior + compatibility result or partial records; and native-validation tests could + not exercise a policy-gated subprocess because that behavior did not exist. +- **Non-blocking environment warnings:** pytest could not write its cache in + the sandboxed feature worktree. + +## Passing-after + +- **Date:** 2026-07-17 (Europe/Berlin) +- **Command:** `hatch run test -- tests/unit/requirements/test_upstream_evidence_imports.py` +- **Result:** 44 passed. +- **Summary:** The pinned v0.12.18 scaffold, incomplete Functional + Requirements, placeholder Functional Requirements, sources with no user + story, scenarios outside a user story, invalid native validator results, + timeouts, and unavailable validators now fail atomically. Enterprise aliases + and layered policy overrides are covered, while portable imports do not probe + an ambient executable. + +## Quality evidence + +- `hatch run format`, `hatch run lint`, `hatch run yaml-lint`, and + `hatch run contract-test`: passed. `hatch run lint` includes the repository + basedpyright type-check gate. +- `hatch run smart-test-full`: 2,869 tests passed and 9 skipped; the local + full-suite line coverage is 64%, above the configured 50% threshold. The + PR runner independently measures 68.57% for its coverage artifact. +- `hatch run smart-test`: focused-change execution passed; it is not used as + the full-suite coverage assertion. +- `hatch run semgrep-sast --json --output logs/static-analysis/semgrep.json`, + `hatch run semgrep-sast-gate --results logs/static-analysis/semgrep.json + --baseline tools/semgrep/sast-baseline.json`, and `hatch run bandit-scan -f + json -o logs/static-analysis/bandit.json`: passed with no blocking findings. +- `hatch run python scripts/pre_commit_code_review.py ...`: passed with zero + findings; report written to `.specfact/code-review.json`. diff --git a/openspec/changes/requirements-04-upstream-source-readiness/design.md b/openspec/changes/requirements-04-upstream-source-readiness/design.md new file mode 100644 index 00000000..e4738d6d --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/design.md @@ -0,0 +1,119 @@ +## Context + +Core native importers currently parse OpenSpec and Spec Kit artifacts directly +into `RequirementContextImportResult`. They already fail closed for unsupported +schemas, but a pristine official Spec Kit template meets the current structural +pattern and emits placeholder requirement records. The modules runtime cannot +solve this safely: it delegates to these core importers and must not own native +parsing, hashes, or gate policy. + +OpenSpec supplies `openspec validate --strict --json` for native validation. +Spec Kit 0.12.15 exposes setup and tool checks but no feature-artifact validator, +so source readiness must inspect its documented native artifact content without +becoming an authoring schema. + +## Goals / Non-Goals + +**Goals:** + +- Return accepted records or zero records with structured error diagnostics. +- Reject known incomplete Spec Kit sources before normalization. +- Support explicit/strict-policy OpenSpec native validation without a mandatory + executable dependency for portable imports. +- Preserve existing valid import behavior, source hashes, idempotency, and + read-only source access. + +**Non-Goals:** + +- Create or require metadata owned by OpenSpec or Spec Kit. +- Write validation results into upstream folders. +- Treat `specify check` as a source validator. +- Detect every semantic ambiguity in prose. +- Move source-readiness policy into the modules repository. + +## Decisions + +### Return one atomic result from existing importers + +`import_openspec_change` and `import_speckit_feature` SHALL evaluate readiness +before returning normalized records. Any error-level readiness diagnostic yields +an empty `RequirementContextImportResult.requirements` collection. Existing +diagnostic fields remain the cross-repository contract used by the Requirements +module. + +This reuses the current result model and prevents a rejected source from being +partially persisted. Filtering records only after normalization is rejected +because placeholder records could leak to any other core caller. + +### Detect only narrow, source-native incompleteness markers for Spec Kit + +The preflight SHALL recognise the official scaffold markers observed in the +supported Spec Kit profile, unresolved `NEEDS CLARIFICATION`, absent substantive +Functional Requirements, and absent meaningful acceptance scenarios when user +stories are present. It SHALL return `incomplete-source-template` for known +template markers and `source-incomplete` for missing required native content. + +Detection uses explicit marker rules plus fixture coverage, not a whole-template +hash or generic bracket matching. This keeps ordinary requirement prose valid +and makes upstream template changes visible through compatibility tests. + +### Gate OpenSpec CLI validation by explicit policy + +The readiness policy SHALL determine whether native OpenSpec validation is +required. When required, core invokes `openspec validate --strict --json` with +bounded process execution and consumes its machine-readable outcome. A failed +validator returns `source-invalid`; a missing executable returns +`upstream-validator-unavailable`. Both are error-level, atomic failures. + +The policy is the layered `validation.openspec.require_native_validation` +boolean. Resolve configuration from organization baseline, repository overlay, +then developer-local overlay; the last explicit boolean wins. If no layer sets +the key, the effective validation tier supplies the default: `enterprise` is +`true`, while `solo`, `startup`, and `mid_size` are `false`. The `strict` and +`enterprise_full_stack` aliases resolve to `enterprise`; `team` and +`api_first_team` resolve to `mid_size`. + +An explicit `profile` argument chooses the tier default but does not discard an +explicit boolean from a configuration layer. Therefore native validation is +mandatory only when the resolved boolean is `true`; otherwise the importer is +portable and must not invoke an ambient executable. + +When policy does not require native validation, importer behavior remains +portable and does not probe an ambient `openspec` executable. Always probing is +rejected because the same source could produce different evidence merely due to +developer PATH state. + +### Keep readiness and downstream evidence gates distinct + +Readiness decides whether a source may become evidence. Existing validation +gates (`scenario-unverified`, `stale-import`, `source-missing`, and +`ambiguous-mapping`) continue to evaluate accepted records after import. +Readiness failures are import diagnostics, not downstream evidence findings. + +## Risks / Trade-offs + +- [Spec Kit template drift] → Pin a current official scaffold fixture and add a + scheduled compatibility test against the supported upstream release. +- [False positives from ordinary bracketed prose] → Detect only narrow known + markers and add a legitimate bracketed-prose regression fixture. +- [Unavailable OpenSpec CLI] → Fail only when policy explicitly requires it; + otherwise preserve portable importer behavior. +- [Subprocess output or timeout variation] → Consume JSON, bound execution, and + surface a stable diagnostic rather than raw tool output as contract data. +- [Core/module release skew] → Publish this core contract before the modules + patch; modules raises its compatibility floor and preserves the diagnostics. + +## Migration Plan + +1. Add core specs, failing tests, and the readiness implementation. +2. Release the core version containing the new contract. +3. Unblock modules issue #346; update module compatibility, command tests, + signature, and patch release. +4. Roll back by pinning the modules runtime to the prior core-compatible module; + readiness never mutates upstream sources or existing accepted sidecars. + +## Compatibility Baseline + +The initial official Spec Kit scaffold fixture is pinned to `v0.12.18`. The +previously observed `v0.12.15` placeholder import remains regression context, +not the supported compatibility baseline. diff --git a/openspec/changes/requirements-04-upstream-source-readiness/proposal.md b/openspec/changes/requirements-04-upstream-source-readiness/proposal.md new file mode 100644 index 00000000..d28ce981 --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/proposal.md @@ -0,0 +1,63 @@ +## Why + +The import-first adapter from `openspec-01-intent-trace` can normalize an +unfinished native source into apparently trustworthy requirement evidence. A +local test of the official Spec Kit 0.12.15 scaffold produced six placeholder +records from its untouched `spec.md`. Invalid OpenSpec changes need the same +protection when repository policy requires native OpenSpec validation. + +Readiness is part of evidence integrity. It must be decided where native +artifacts are parsed and normalized, before a module can persist a sidecar or +CI can consume coverage findings. + +## What Changes + +- Add a core-owned, atomic source-readiness preflight to native OpenSpec and + Spec Kit requirement imports. +- Reject known incomplete Spec Kit sources with structured diagnostics and zero + normalized records: official draft placeholders, unresolved + `NEEDS CLARIFICATION`, no substantive Functional Requirement, or no meaningful + acceptance scenario when user stories exist. +- Under explicit or strict/enterprise upstream-validation policy, invoke + `openspec validate --strict --json` for the selected change; reject failed + validation or an unavailable required validator with named diagnostics. +- Preserve portable basic OpenSpec import when policy does not require the + OpenSpec CLI, without ambient executable probing or a new hard dependency. +- Preserve completed-source stable IDs, SHA-256 provenance, idempotency, and + read-only behavior. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `openspec-speckit-evidence-adapter`: Native import gains deterministic + source-readiness outcomes before records are emitted. + +## Impact + +- Affected core code: `specfact_cli.requirements.importers`, requirements + import result/diagnostic contracts, and upstream import tests. +- Affected modules runtime: `nold-ai/specfact-cli-modules#346` consumes the + resulting diagnostics and prevents persistence for rejected sources. +- No upstream authoring schema, upstream write-back, or Spec Kit CLI dependency + is introduced. +- Compatibility impact: the paired Requirements module must raise its minimum + core compatibility version after this core release; existing accepted native + sources remain backward-compatible. + +--- + +## Source Tracking + + +- **GitHub Issue**: #648 +- **Issue URL**: +- **Parent Feature**: #366 Requirements Layer +- **Follow-up To**: #350 +- **Paired Modules Issue**: nold-ai/specfact-cli-modules#346 +- **Last Synced Status**: open / Todo (aligned 2026-07-14) +- **Sanitized**: false diff --git a/openspec/changes/requirements-04-upstream-source-readiness/specs/openspec-speckit-evidence-adapter/spec.md b/openspec/changes/requirements-04-upstream-source-readiness/specs/openspec-speckit-evidence-adapter/spec.md new file mode 100644 index 00000000..aaddac34 --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/specs/openspec-speckit-evidence-adapter/spec.md @@ -0,0 +1,107 @@ +## ADDED Requirements + +### Requirement: Native source readiness before requirement normalization + +The OpenSpec and Spec Kit evidence adapter SHALL evaluate native source readiness +before emitting `RequirementInput` records. If readiness returns any error-level +diagnostic, the adapter SHALL return zero records and SHALL preserve the source +directory byte-for-byte. Readiness diagnostics SHALL use the existing structured +import-result contract and SHALL distinguish incomplete Spec Kit sources, +upstream-invalid OpenSpec sources, and required unavailable validators. + +For the supported Spec Kit profile, a scaffold marker takes precedence over all +other content and returns `incomplete-source-template`. A marker is any pinned +official scaffold literal recognized by the adapter or the literal +`[NEEDS CLARIFICATION:`. Without a scaffold marker, a substantive Functional +Requirement is a supported `FR-` entry whose text after `System MUST` is +non-empty and contains no `[` or `]` placeholder delimiter. A meaningful +acceptance scenario is a parsed GIVEN/WHEN/THEN rule within the Markdown block +that starts with a `### User Story - ` heading and ends at the +next heading of level one through three. A source lacking either required +element returns `source-incomplete`; a scenario outside such a story block does +not satisfy readiness. The pinned fixture mapping is: the byte-identical +`v0.12.18` scaffold is `incomplete-source-template`; a completed fixture is +accepted; and fixtures lacking a Functional Requirement or a story scenario are +`source-incomplete`. Core and module #346 SHALL use these same diagnostics. + +#### Scenario: Reject a pristine Spec Kit scaffold + +- **GIVEN** a supported native Spec Kit feature created from the official + scaffold that retains recognised draft placeholders or + `NEEDS CLARIFICATION` markers +- **WHEN** the adapter imports the feature +- **THEN** it returns zero requirement records +- **AND** it returns an error diagnostic with code + `incomplete-source-template` and the relevant source locator +- **AND** the feature directory remains byte-identical. + +#### Scenario: Reject a structurally incomplete Spec Kit source + +- **GIVEN** a native Spec Kit source with no substantive Functional Requirement, + no recognized user-story block, or no meaningful acceptance scenario within a + recognized user-story block +- **WHEN** the adapter imports the feature +- **THEN** it returns zero requirement records +- **AND** it returns an error diagnostic with code `source-incomplete`. + +#### Scenario: Import a completed native Spec Kit source + +- **GIVEN** a native Spec Kit feature with substantive Functional Requirements + and meaningful GIVEN/WHEN/THEN acceptance scenarios +- **WHEN** the adapter imports the feature +- **THEN** it returns normalized records with stable derived IDs and SHA-256 + source revisions +- **AND** re-importing the unchanged source returns the same result without + changing the source directory. + +#### Scenario: Reject invalid OpenSpec under required native validation + +- **GIVEN** source-readiness policy requires native OpenSpec validation and + `openspec validate --strict --json` reports the selected change invalid +- **WHEN** the adapter imports the OpenSpec change +- **THEN** it returns zero requirement records +- **AND** it returns an error diagnostic with code `source-invalid` +- **AND** it does not fall back to import that claims native validation passed. + +#### Scenario: Reject failed or unusable native validation output + +- **GIVEN** required native OpenSpec validation times out, exits non-zero, emits + malformed or empty JSON, or exits zero without a passing result item for the + selected change +- **WHEN** the adapter imports the OpenSpec change +- **THEN** it returns zero requirement records +- **AND** it returns an error diagnostic with code `source-invalid` +- **AND** the change directory remains byte-identical. + +#### Scenario: Report an unavailable required OpenSpec validator + +- **GIVEN** source-readiness policy requires native OpenSpec validation and the + OpenSpec CLI is unavailable +- **WHEN** the adapter imports an OpenSpec change +- **THEN** it returns zero requirement records +- **AND** it returns an error diagnostic with code + `upstream-validator-unavailable`. + +#### Scenario: Require native OpenSpec validation for the enterprise tier + +- **GIVEN** the effective requirements profile is `enterprise` and no layered + configuration overrides its default +- **WHEN** the adapter imports an OpenSpec change +- **THEN** it requires native OpenSpec validation before emitting records. + +#### Scenario: Resolve native validation policy through aliases and overrides + +- **GIVEN** `strict` or `enterprise_full_stack` is the effective profile with + no explicit policy value +- **WHEN** the adapter imports an OpenSpec change +- **THEN** it requires native OpenSpec validation +- **AND** an explicit layered boolean overrides that profile default. + +#### Scenario: Preserve portable OpenSpec import without required validation + +- **GIVEN** source-readiness policy does not require native OpenSpec validation + and the source satisfies the supported core schema profile +- **WHEN** the adapter imports the OpenSpec change on a host without the + OpenSpec CLI +- **THEN** it preserves the existing normalized import behavior +- **AND** it does not claim native CLI validation occurred. diff --git a/openspec/changes/requirements-04-upstream-source-readiness/tasks.md b/openspec/changes/requirements-04-upstream-source-readiness/tasks.md new file mode 100644 index 00000000..0f464416 --- /dev/null +++ b/openspec/changes/requirements-04-upstream-source-readiness/tasks.md @@ -0,0 +1,59 @@ +## 1. Governance and dependency readiness + +- [x] 1.1 Create core issue #648 under Requirements Layer feature #366 with + `enhancement` and `change-proposal` labels and SpecFact CLI project assignment. +- [x] 1.2 Link the modules companion issue + `nold-ai/specfact-cli-modules#346` in the paired proposal and issue scope. +- [ ] 1.3 Synchronize #648 and #346 with the released core version and explicit + cross-repository blocker relation before module implementation begins. +- [ ] 1.4 Recheck issue state, parent, labels, project assignment, blockers, + and active-work concurrency immediately before implementation. + +## 2. Specification and failing evidence + +- [x] 2.1 Add a pinned fixture representing the supported official Spec Kit + scaffold and derive failing tests for placeholder and + `NEEDS CLARIFICATION` rejection. +- [x] 2.2 Add failing tests for missing substantive Functional Requirements and + missing meaningful acceptance scenarios while user stories are present. +- [x] 2.3 Add failing tests proving completed Spec Kit sources retain stable IDs, + SHA-256 revisions, given/when/then rules, idempotency, and read-only sources. +- [x] 2.4 Add failing tests for policy-required OpenSpec validation failure, + missing required validator, and portable no-validator import. +- [x] 2.5 Record targeted failing-before evidence in `TDD_EVIDENCE.md`. + +## 3. Core source-readiness implementation + +- [x] 3.1 Define the contract-safe readiness policy and bounded OpenSpec CLI + invocation without ambient executable probing. +- [x] 3.2 Implement atomic readiness diagnostics in the native import-result + contract: `incomplete-source-template`, `source-incomplete`, + `source-invalid`, and `upstream-validator-unavailable`. +- [x] 3.3 Implement narrow, fixture-backed Spec Kit draft-marker and structural + completeness checks without generic bracket matching or a new authoring + schema. +- [x] 3.4 Integrate readiness before OpenSpec and Spec Kit normalization, with + public API contracts (`@beartype`, `@require`, `@ensure`) on any new public + surfaces. +- [x] 3.5 Run targeted tests and record passing-after evidence in + `TDD_EVIDENCE.md`. + +## 4. Compatibility, documentation, and quality gates + +- [x] 4.1 Add regression coverage for existing accepted native imports and + readiness-versus-downstream-gate separation. +- [x] 4.2 Update core requirements/import documentation with source-readiness + diagnostics and policy behavior; cross-link the modules command guidance. +- [x] 4.3 Run format, type-check, lint, contract, smart-test, targeted tests, + and relevant documentation checks. +- [x] 4.4 Run a fresh full SpecFact code review JSON report after the last + substantive change, resolve every finding, and record the evidence. + +## 5. Delivery + +- [ ] 5.1 Run `openspec validate requirements-04-upstream-source-readiness --strict` + and synchronize #648 and the internal wiki source page with final scope, + version, dependencies, and status. +- [ ] 5.2 Release the core contract, update the modules issue blocker, and open + the companion modules implementation only after compatibility is published. +- [ ] 5.3 Open the core PR to `dev` with core and modules dependency evidence. diff --git a/pyproject.toml b/pyproject.toml index 284eff85..5045c221 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "specfact-cli" -version = "0.52.3" +version = "0.53.1" description = "AI-bloat defense CLI for Python teams. Run deterministic code review, cleanup forecasts, and spec/contract evidence for AI-assisted and brownfield delivery." readme = "README.md" requires-python = ">=3.11" diff --git a/setup.py b/setup.py index 7557ab37..e7d5db1e 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ if __name__ == "__main__": _setup = setup( name="specfact-cli", - version="0.52.3", + version="0.53.1", description=( "AI-bloat defense CLI for Python teams. Run deterministic code review, cleanup forecasts, " "and spec/contract evidence for AI-assisted and brownfield delivery." diff --git a/src/__init__.py b/src/__init__.py index c50dba19..e248ee12 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -3,4 +3,4 @@ """ # Package version: keep in sync with pyproject.toml, setup.py, src/specfact_cli/__init__.py -__version__ = "0.52.3" +__version__ = "0.53.1" diff --git a/src/specfact_cli/__init__.py b/src/specfact_cli/__init__.py index 20e3a57d..62f3882d 100644 --- a/src/specfact_cli/__init__.py +++ b/src/specfact_cli/__init__.py @@ -76,6 +76,6 @@ def _install_progressive_disclosure() -> None: # keeps missing-command and missing-parameter UX consistent outside the root CLI too. _install_progressive_disclosure() -__version__ = "0.52.3" +__version__ = "0.53.1" __all__ = ["__version__"] diff --git a/src/specfact_cli/requirements/__init__.py b/src/specfact_cli/requirements/__init__.py index cca77ecd..3cbdde1f 100644 --- a/src/specfact_cli/requirements/__init__.py +++ b/src/specfact_cli/requirements/__init__.py @@ -10,6 +10,7 @@ inspect_requirement_context_coverage, load_requirements_from_bundle, normalize_requirement_records, + requires_native_openspec_validation, validate_requirement_context, ) from specfact_cli.requirements.importers import import_openspec_change, import_speckit_feature @@ -27,5 +28,6 @@ "inspect_requirement_context_coverage", "load_requirements_from_bundle", "normalize_requirement_records", + "requires_native_openspec_validation", "validate_requirement_context", ] diff --git a/src/specfact_cli/requirements/context.py b/src/specfact_cli/requirements/context.py index c6fc714e..63045172 100644 --- a/src/specfact_cli/requirements/context.py +++ b/src/specfact_cli/requirements/context.py @@ -215,6 +215,30 @@ def _resolve_requirement_profile( return effective_profile, resolved.values +@beartype +@require(_optional_profile_supported, "profile must be a supported requirements profile when provided") +@require(_project_root_supported, "project_root must be a Path when provided") +@ensure(lambda result: isinstance(result, bool), "result must be a boolean") +def requires_native_openspec_validation( + *, + profile: RequirementContextValidationProfile | None = None, + project_root: Path | None = None, +) -> bool: + """Return whether layered policy requires native OpenSpec validation.""" + effective_profile, resolved_config = _resolve_requirement_profile(profile, project_root) + validation_tier = _PROFILE_RESOLUTION_ALIASES.get(effective_profile, effective_profile) + validation_value = resolved_config.get("validation") + if not isinstance(validation_value, Mapping): + return validation_tier == "enterprise" + validation = cast(Mapping[str, object], validation_value) + openspec_value = validation.get("openspec") + if not isinstance(openspec_value, Mapping): + return validation_tier == "enterprise" + openspec = cast(Mapping[str, object], openspec_value) + configured = openspec.get("require_native_validation") + return configured if isinstance(configured, bool) else validation_tier == "enterprise" + + def _is_imported_requirement(requirement: RequirementInput) -> bool: return any( source.source_type in {RequirementSourceType.OPENSPEC_CHANGE, RequirementSourceType.SPECKIT_SPEC} diff --git a/src/specfact_cli/requirements/importers.py b/src/specfact_cli/requirements/importers.py index 6c46ede5..8cbe8d6b 100644 --- a/src/specfact_cli/requirements/importers.py +++ b/src/specfact_cli/requirements/importers.py @@ -3,7 +3,9 @@ from __future__ import annotations import hashlib +import json import re +import subprocess from pathlib import Path from typing import cast @@ -23,6 +25,8 @@ RequirementContextDiagnostic, RequirementContextDiagnosticSeverity, RequirementContextImportResult, + RequirementContextValidationProfile, + requires_native_openspec_validation, ) @@ -30,12 +34,24 @@ _OPENSPEC_REQUIREMENT_PATTERN = re.compile(r"^### Requirement:\s+.+$", re.MULTILINE) _OPENSPEC_SCENARIO_PATTERN = re.compile(r"^#### Scenario:\s+.+$", re.MULTILINE) _SPECKIT_TITLE_PATTERN = re.compile(r"^# Feature Specification:\s+.+$", re.MULTILINE) -_SPECKIT_REQUIREMENT_PATTERN = re.compile(r"^\s*-?\s*\*\*FR-\d+\*\*:\s*System MUST\s+.+$", re.MULTILINE) +_SPECKIT_REQUIREMENT_PATTERN = re.compile(r"^\s*-?\s*\*\*FR-\d+\*\*:\s*System MUST\s+(?P<text>.+?)\s*$", re.MULTILINE) _SPECKIT_CUSTOMIZATION_ROOTS = ( Path(".specify/templates/overrides"), Path(".specify/presets"), Path(".specify/extensions"), ) +_OPENSPEC_VALIDATOR_TIMEOUT_SECONDS = 10 +_SPECKIT_TEMPLATE_MARKERS = ( + "# Feature Specification: [FEATURE NAME]", + "**Feature Branch**: `[###-feature-name]`", + '**Input**: User description: "$ARGUMENTS"', + "### User Story 1 - [Brief Title] (Priority: P1)", + "[Describe this user journey in plain language]", + "- **FR-001**: System MUST [specific capability,", +) +_SPECKIT_NEEDS_CLARIFICATION_MARKER = "[NEEDS CLARIFICATION:" +_SPECKIT_USER_STORY_PATTERN = re.compile(r"^### User Story\s+\d+\s+-\s+.+$", re.MULTILINE) +_SPECKIT_SECTION_PATTERN = re.compile(r"^#{1,3}\s+", re.MULTILINE) def _slug(value: str) -> str: @@ -87,13 +103,33 @@ def _scenario_rules(requirement_id: str, content: str) -> list[BusinessRule]: return rules +def _speckit_user_story_blocks(content: str) -> list[str]: + """Return the content of explicitly headed Spec Kit user stories.""" + matches = list(_SPECKIT_USER_STORY_PATTERN.finditer(content)) + blocks: list[str] = [] + for match in matches: + following_section = _SPECKIT_SECTION_PATTERN.search(content, match.end()) + end = following_section.start() if following_section else len(content) + blocks.append(content[match.start() : end]) + return blocks + + +def _speckit_has_substantive_requirement(content: str) -> bool: + """Return whether a supported FR entry contains completed, non-template text.""" + return any( + "[" not in match.group("text") and "]" not in match.group("text") + for match in _SPECKIT_REQUIREMENT_PATTERN.finditer(content) + ) + + def _speckit_rules(requirement_id: str, content: str) -> list[BusinessRule]: rules: list[BusinessRule] = [] pattern = re.compile( r"\*\*Given\*\*\s+(.+?),\s*\*\*When\*\*\s+(.+?),\s*\*\*Then\*\*\s+(.+?)(?=\n|$)", re.IGNORECASE, ) - for index, match in enumerate(pattern.finditer(content), start=1): + story_content = "\n".join(_speckit_user_story_blocks(content)) + for index, match in enumerate(pattern.finditer(story_content), start=1): rules.append( BusinessRule( rule_id=f"{requirement_id}:scenario-{index}", @@ -136,6 +172,24 @@ def _unsupported_source_result( ) +def _readiness_error_result( + source: Path, + source_type: RequirementSourceType, + code: str, + message: str, +) -> RequirementContextImportResult: + return RequirementContextImportResult( + diagnostics=[ + RequirementContextDiagnostic( + severity=RequirementContextDiagnosticSeverity.ERROR, + code=code, + message=message, + source_locator=str(source), + ) + ] + ) + + def _load_schema_name(path: Path) -> str | None: try: if not path.is_file(): @@ -205,18 +259,123 @@ def _speckit_compatibility_error( return None +def _speckit_readiness_error(spec_path: Path, content: str) -> RequirementContextImportResult | None: + if not _SPECKIT_TITLE_PATTERN.search(content): + return None + if any(marker in content for marker in _SPECKIT_TEMPLATE_MARKERS) or _SPECKIT_NEEDS_CLARIFICATION_MARKER in content: + return _readiness_error_result( + spec_path, + RequirementSourceType.SPECKIT_SPEC, + "incomplete-source-template", + "Spec Kit source still contains a supported official scaffold placeholder.", + ) + if not _speckit_has_substantive_requirement(content): + return _readiness_error_result( + spec_path, + RequirementSourceType.SPECKIT_SPEC, + "source-incomplete", + "Spec Kit source has no substantive Functional Requirement.", + ) + if not _speckit_rules("readiness", content): + return _readiness_error_result( + spec_path, + RequirementSourceType.SPECKIT_SPEC, + "source-incomplete", + "Spec Kit source has no recognized user-story acceptance scenario.", + ) + return None + + +def _openspec_native_validation_error( + change_dir: Path, + *, + profile: RequirementContextValidationProfile | None, + project_root: Path | None, +) -> RequirementContextImportResult | None: + if not requires_native_openspec_validation(profile=profile, project_root=project_root): + return None + try: + completed = subprocess.run( + ["openspec", "validate", change_dir.name, "--strict", "--json"], + cwd=change_dir.parents[2], + capture_output=True, + check=False, + text=True, + timeout=_OPENSPEC_VALIDATOR_TIMEOUT_SECONDS, + ) + except FileNotFoundError: + return _readiness_error_result( + change_dir, + RequirementSourceType.OPENSPEC_CHANGE, + "upstream-validator-unavailable", + "Required native OpenSpec validator is unavailable.", + ) + except (OSError, subprocess.TimeoutExpired): + return _readiness_error_result( + change_dir, + RequirementSourceType.OPENSPEC_CHANGE, + "source-invalid", + "Required native OpenSpec validation did not complete successfully.", + ) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError: + payload = None + if completed.returncode != 0 or not _native_openspec_result_valid(payload, change_dir.name): + return _readiness_error_result( + change_dir, + RequirementSourceType.OPENSPEC_CHANGE, + "source-invalid", + "Required native OpenSpec validation reported the source invalid.", + ) + return None + + +def _native_openspec_result_valid(payload: object, change_name: str) -> bool: + if not isinstance(payload, dict): + return False + result = cast(dict[str, object], payload) + items = result.get("items") + if not isinstance(items, list): + return False + return any( + isinstance(item, dict) + and cast(dict[str, object], item).get("id") == change_name + and cast(dict[str, object], item).get("valid") is True + for item in items + ) + + def _import_result_has_requirements(result: RequirementContextImportResult) -> bool: return all(isinstance(record, RequirementInput) for record in result.requirements) @require(lambda change_dir: isinstance(change_dir, Path), "change_dir must be a Path") +@require(lambda profile: profile is None or isinstance(profile, str), "profile must be text when provided") +@require( + lambda project_root: project_root is None or isinstance(project_root, Path), + "project_root must be a Path when provided", +) @ensure(_import_result_has_requirements) @beartype -def import_openspec_change(change_dir: Path) -> RequirementContextImportResult: +def import_openspec_change( + change_dir: Path, + *, + profile: RequirementContextValidationProfile | None = None, + project_root: Path | None = None, +) -> RequirementContextImportResult: """Normalize one native OpenSpec change directory without modifying it.""" if not change_dir.is_dir(): return _missing_source_result(change_dir, RequirementSourceType.OPENSPEC_CHANGE) + native_validation_error = _openspec_native_validation_error( + change_dir, + profile=profile, + project_root=project_root, + ) + if native_validation_error: + return native_validation_error + spec_paths = sorted((change_dir / "specs").glob("*/spec.md")) compatibility_error = _openspec_compatibility_error(change_dir, spec_paths) if compatibility_error: @@ -279,6 +438,9 @@ def import_speckit_feature(feature_dir: Path) -> RequirementContextImportResult: return _missing_source_result(feature_dir, RequirementSourceType.SPECKIT_SPEC) content = spec_path.read_text(encoding="utf-8") + readiness_error = _speckit_readiness_error(spec_path, content) + if readiness_error: + return readiness_error compatibility_error = _speckit_compatibility_error(feature_dir, spec_path, content) if compatibility_error: return compatibility_error diff --git a/tests/fixtures/speckit/spec-template-v0.12.18.md b/tests/fixtures/speckit/spec-template-v0.12.18.md new file mode 100644 index 00000000..ceb28776 --- /dev/null +++ b/tests/fixtures/speckit/spec-template-v0.12.18.md @@ -0,0 +1,131 @@ +# Feature Specification: [FEATURE NAME] + +**Feature Branch**: `[###-feature-name]` + +**Created**: [DATE] + +**Status**: Draft + +**Input**: User description: "$ARGUMENTS" + +## User Scenarios & Testing *(mandatory)* + +<!-- + IMPORTANT: User stories should be PRIORITIZED as user journeys ordered by importance. + Each user story/journey must be INDEPENDENTLY TESTABLE - meaning if you implement just ONE of them, + you should still have a viable MVP (Minimum Viable Product) that delivers value. + + Assign priorities (P1, P2, P3, etc.) to each story, where P1 is the most critical. + Think of each story as a standalone slice of functionality that can be: + - Developed independently + - Tested independently + - Deployed independently + - Demonstrated to users independently +--> + +### User Story 1 - [Brief Title] (Priority: P1) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently - e.g., "Can be fully tested by [specific action] and delivers [specific value]"] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] +2. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 2 - [Brief Title] (Priority: P2) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 3 - [Brief Title] (Priority: P3) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +[Add more user stories as needed, each with an assigned priority] + +### Edge Cases + +<!-- + ACTION REQUIRED: The content in this section represents placeholders. + Fill them out with the right edge cases. +--> + +- What happens when [boundary condition]? +- How does system handle [error scenario]? + +## Requirements *(mandatory)* + +<!-- + ACTION REQUIRED: The content in this section represents placeholders. + Fill them out with the right functional requirements. +--> + +### Functional Requirements + +- **FR-001**: System MUST [specific capability, e.g., "allow users to create accounts"] +- **FR-002**: System MUST [specific capability, e.g., "validate email addresses"] +- **FR-003**: Users MUST be able to [key interaction, e.g., "reset their password"] +- **FR-004**: System MUST [data requirement, e.g., "persist user preferences"] +- **FR-005**: System MUST [behavior, e.g., "log all security events"] + +*Example of marking unclear requirements:* + +- **FR-006**: System MUST authenticate users via [NEEDS CLARIFICATION: auth method not specified - email/password, SSO, OAuth?] +- **FR-007**: System MUST retain user data for [NEEDS CLARIFICATION: retention period not specified] + +### Key Entities *(include if feature involves data)* + +- **[Entity 1]**: [What it represents, key attributes without implementation] +- **[Entity 2]**: [What it represents, relationships to other entities] + +## Success Criteria *(mandatory)* + +<!-- + ACTION REQUIRED: Define measurable success criteria. + These must be technology-agnostic and measurable. +--> + +### Measurable Outcomes + +- **SC-001**: [Measurable metric, e.g., "Users can complete account creation in under 2 minutes"] +- **SC-002**: [Measurable metric, e.g., "System handles 1000 concurrent users without degradation"] +- **SC-003**: [User satisfaction metric, e.g., "90% of users successfully complete primary task on first attempt"] +- **SC-004**: [Business metric, e.g., "Reduce support tickets related to [X] by 50%"] + +## Assumptions + +<!-- + ACTION REQUIRED: The content in this section represents placeholders. + Fill them out with the right assumptions based on reasonable defaults + chosen when the feature description did not specify certain details. +--> + +- [Assumption about target users, e.g., "Users have stable internet connectivity"] +- [Assumption about scope boundaries, e.g., "Mobile support is out of scope for v1"] +- [Assumption about data/environment, e.g., "Existing authentication system will be reused"] +- [Dependency on existing system/service, e.g., "Requires access to the existing user profile API"] diff --git a/tests/unit/requirements/test_upstream_evidence_imports.py b/tests/unit/requirements/test_upstream_evidence_imports.py index 76641e90..2c156504 100644 --- a/tests/unit/requirements/test_upstream_evidence_imports.py +++ b/tests/unit/requirements/test_upstream_evidence_imports.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import subprocess from pathlib import Path import pytest @@ -24,6 +25,7 @@ import_speckit_feature, validate_requirement_context, ) +from specfact_cli.requirements.context import RequirementContextValidationProfile def _bundle() -> ProjectBundle: @@ -153,6 +155,235 @@ def test_import_speckit_feature_normalizes_requirement_and_scenario(tmp_path: Pa assert requirement.business_rules[0].then == "the widget is returned" +def test_import_speckit_feature_rejects_pristine_v01218_scaffold(tmp_path: Path) -> None: + """The pinned official Spec Kit scaffold cannot produce placeholder evidence.""" + feature_dir = tmp_path / "specs" / "001-widget-rendering" + spec_file = feature_dir / "spec.md" + feature_dir.mkdir(parents=True) + fixture = Path(__file__).parents[2] / "fixtures" / "speckit" / "spec-template-v0.12.18.md" + spec_file.write_bytes(fixture.read_bytes()) + before = spec_file.read_bytes() + + result = import_speckit_feature(feature_dir) + + assert result.requirements == [] + assert [(diagnostic.code, diagnostic.severity) for diagnostic in result.diagnostics] == [ + ("incomplete-source-template", "error") + ] + assert result.diagnostics[0].source_locator == str(spec_file) + assert spec_file.read_bytes() == before + + +@pytest.mark.parametrize( + "source", + [ + """# Feature Specification: Widget rendering + +## Requirements + +### Functional Requirements +""", + """# Feature Specification: Widget rendering + +## User Scenarios & Testing + +### User Story 1 - Render widgets (Priority: P1) + +As a user, I want widgets rendered so that I can see them. + +## Requirements + +### Functional Requirements + +- **FR-001**: System MUST render a widget +""", + """# Feature Specification: Widget rendering + +## Requirements + +- **FR-001**: System MUST render a widget + +## Acceptance Scenarios + +1. **Given** a valid widget request, **When** rendering runs, **Then** the widget is returned +""", + """# Feature Specification: Widget rendering + +## User Scenarios & Testing + +### User Story 1 - Render widgets (Priority: P1) + +As a user, I want widgets rendered so that I can see them. + +## Requirements + +- **FR-001**: System MUST render a widget + +## Acceptance Scenarios + +1. **Given** a valid widget request, **When** rendering runs, **Then** the widget is returned +""", + """# Feature Specification: Widget rendering + +## User Scenarios & Testing + +### User Story 1 - Render widgets (Priority: P1) + +As a user, I want widgets rendered so that I can see them. + +**Acceptance Scenarios**: + +1. **Given** a valid widget request, **When** rendering runs, **Then** the widget is returned + +## Requirements + +- **FR-001**: System MUST [describe the widget behavior] +""", + ], + ids=[ + "missing-functional-requirement", + "story-without-acceptance-scenario", + "no-user-story", + "scenario-outside-user-story", + "placeholder-functional-requirement", + ], +) +def test_import_speckit_feature_rejects_structurally_incomplete_source(tmp_path: Path, source: str) -> None: + """Sources missing Functional Requirements or story scenarios fail atomically.""" + feature_dir = tmp_path / "specs" / "001-widget-rendering" + spec_file = feature_dir / "spec.md" + feature_dir.mkdir(parents=True) + spec_file.write_text(source, encoding="utf-8") + before = spec_file.read_bytes() + + result = import_speckit_feature(feature_dir) + + assert result.requirements == [] + assert [(diagnostic.code, diagnostic.severity) for diagnostic in result.diagnostics] == [ + ("source-incomplete", "error") + ] + assert spec_file.read_bytes() == before + + +@pytest.mark.parametrize( + ("returncode", "stdout"), + [(1, '{"valid": false}'), (0, '{"valid": false}'), (0, "not-json"), (0, "")], + ids=["validator-failure", "zero-exit-invalid-item", "malformed-json", "empty-json"], +) +@pytest.mark.parametrize("profile", ["enterprise", "strict", "enterprise_full_stack"]) +def test_import_openspec_change_rejects_invalid_required_native_validation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + returncode: int, + stdout: str, + profile: RequirementContextValidationProfile, +) -> None: + """Required-profile imports reject failed or unusable native validation results.""" + change_dir = tmp_path / "openspec" / "changes" / "widget-evidence" + spec_file = _write_openspec_change(change_dir) + before = spec_file.read_bytes() + monkeypatch.setattr( + "specfact_cli.requirements.importers.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout), + ) + + result = import_openspec_change(change_dir, profile=profile, project_root=tmp_path) + + assert result.requirements == [] + assert [(diagnostic.code, diagnostic.severity) for diagnostic in result.diagnostics] == [ + ("source-invalid", "error") + ] + assert spec_file.read_bytes() == before + + +def test_import_openspec_change_rejects_timed_out_required_native_validation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A required native validation timeout fails closed without changing the source.""" + change_dir = tmp_path / "openspec" / "changes" / "widget-evidence" + spec_file = _write_openspec_change(change_dir) + before = spec_file.read_bytes() + monkeypatch.setattr( + "specfact_cli.requirements.importers.subprocess.run", + lambda *_args, **_kwargs: (_ for _ in ()).throw(subprocess.TimeoutExpired(cmd="openspec", timeout=10)), + ) + + result = import_openspec_change(change_dir, profile="enterprise", project_root=tmp_path) + + assert result.requirements == [] + assert [(diagnostic.code, diagnostic.severity) for diagnostic in result.diagnostics] == [ + ("source-invalid", "error") + ] + assert spec_file.read_bytes() == before + + +def test_import_openspec_change_reports_missing_required_native_validator( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Enterprise imports name a missing required OpenSpec executable.""" + change_dir = tmp_path / "openspec" / "changes" / "widget-evidence" + _write_openspec_change(change_dir) + monkeypatch.setattr( + "specfact_cli.requirements.importers.subprocess.run", + lambda *_args, **_kwargs: (_ for _ in ()).throw(FileNotFoundError()), + ) + + result = import_openspec_change(change_dir, profile="enterprise", project_root=tmp_path) + + assert result.requirements == [] + assert [(diagnostic.code, diagnostic.severity) for diagnostic in result.diagnostics] == [ + ("upstream-validator-unavailable", "error") + ] + + +def test_import_openspec_change_honors_layered_native_validation_override( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A repository policy can require validation outside the enterprise tier.""" + change_dir = tmp_path / "openspec" / "changes" / "widget-evidence" + _write_openspec_change(change_dir) + config_dir = tmp_path / ".specfact" + config_dir.mkdir() + (config_dir / "config.yaml").write_text( + """profile: startup +validation: + openspec: + require_native_validation: true +""", + encoding="utf-8", + ) + monkeypatch.setattr( + "specfact_cli.requirements.importers.subprocess.run", + lambda *_args, **_kwargs: subprocess.CompletedProcess(args=[], returncode=1, stdout='{"valid": false}'), + ) + + result = import_openspec_change(change_dir, project_root=tmp_path) + + assert result.requirements == [] + assert result.diagnostics[0].code == "source-invalid" + + +def test_import_openspec_change_does_not_probe_native_validator_when_not_required( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Portable imports do not depend on an ambient OpenSpec executable.""" + change_dir = tmp_path / "openspec" / "changes" / "widget-evidence" + _write_openspec_change(change_dir) + monkeypatch.setattr( + "specfact_cli.requirements.importers.subprocess.run", + lambda *_args, **_kwargs: pytest.fail("portable import must not probe openspec"), + ) + + result = import_openspec_change(change_dir, profile="startup", project_root=tmp_path) + + assert len(result.requirements) == 1 + assert result.diagnostics == [] + + def test_import_speckit_feature_reports_malformed_requirement_entries( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/workflows/test_trustworthy_green_checks.py b/tests/unit/workflows/test_trustworthy_green_checks.py index ead5b5fd..ef97daff 100644 --- a/tests/unit/workflows/test_trustworthy_green_checks.py +++ b/tests/unit/workflows/test_trustworthy_green_checks.py @@ -212,7 +212,9 @@ def test_pr_orchestrator_quality_gates_are_blocking() -> None: run_blocks = "\n".join(str(step.get("run", "")) for step in _load_job_steps("quality-gates")) assert "advisory" not in run_blocks.lower() assert "fail_under" in run_blocks or "COVERAGE_FAIL_UNDER" in run_blocks - assert "max(configured_fail_under, 80.0)" in run_blocks + assert "print(configured_fail_under)" in run_blocks + assert "max(configured_fail_under, 80.0)" not in run_blocks + assert "set -o pipefail" in run_blocks assert "exit 1" in run_blocks