Require exhaustive Python behavior parity before Go cutover

.crane/scripts/score.go

@@ -36,7 +36,12 @@
 //   Gate 8 -- benchmarks_pass: TestParityCompletionBenchmarks must pass.
 //             Migration benchmarks must run and satisfy the configured guard.
 //
-//   Gate 9 -- no_known_exceptions: the test output must not contain any
+//   Gate 9 -- python_behavior_contracts:
+//             TestParityCompletionPythonBehaviorContracts must pass. Every
+//             extracted Python command and existing Python test must be mapped
+//             to Go tests and CLI-agnostic parity tests.
+//
+//   Gate 10 -- no_known_exceptions: the test output must not contain any
 //             "approved exception" log line. Final cutover requires zero exceptions.
 //
 // If Gate 1 fails, migration_score is forced to 0.0 regardless of other gates.
@@ -93,6 +98,7 @@ func main() {
 		gateStateDiffContracts  = "TestParityCompletionStateDiffContracts"
 		gatePythonSuite         = "TestParityCompletionPythonSuite"
 		gateBenchmarks          = "TestParityCompletionBenchmarks"
+		gateBehaviorContracts   = "TestParityCompletionPythonBehaviorContracts"
 	)
 
 	// Track per-test pass/fail.
@@ -187,16 +193,19 @@ func main() {
 	// Gate 8: benchmarks_pass
 	gate8 := singleTestGate("benchmarks_pass", gateBenchmarks, testPassed, testFailed)
 
-	// Gate 9: no_known_exceptions
-	gate9 := GateResult{Name: "no_known_exceptions"}
+	// Gate 9: python_behavior_contracts
+	gate9 := singleTestGate("python_behavior_contracts", gateBehaviorContracts, testPassed, testFailed)
+
+	// Gate 10: no_known_exceptions
+	gate10 := GateResult{Name: "no_known_exceptions"}
 	if knownExceptionsFound {
-		gate9.Passing = false
-		gate9.Reason = "output contains 'approved exception' -- all exceptions must be resolved for cutover"
+		gate10.Passing = false
+		gate10.Reason = "output contains 'approved exception' -- all exceptions must be resolved for cutover"
 	} else {
-		gate9.Passing = true
+		gate10.Passing = true
 	}
 
-	gates := []GateResult{gate1, gate2, gate3, gate4, gate5, gate6, gate7, gate8, gate9}
+	gates := []GateResult{gate1, gate2, gate3, gate4, gate5, gate6, gate7, gate8, gate9, gate10}
 
 	// Count parity tests (any test with "Parity" in name from cmd/apm).
 	parityPassing, parityTotal := 0, 0

.github/workflows/migration-ci.yml

@@ -45,7 +45,7 @@ jobs:
             "${{ github.event.pull_request.head.sha }}" \
             | tee "$RUNNER_TEMP/changed-files.txt"
 
-          if grep -Eq '^(\.crane/|\.github/workflows/migration-ci\.yml$|cmd/|internal/|pkg/|go\.mod$|go\.sum$|pyproject\.toml$|scripts/ci/|src/|tests/benchmarks/|tests/unit/test_crane_score\.py$)' "$RUNNER_TEMP/changed-files.txt"; then
+          if grep -Eq '^(\.crane/|\.github/workflows/migration-ci\.yml$|cmd/|internal/|pkg/|go\.mod$|go\.sum$|pyproject\.toml$|scripts/ci/|src/|tests/benchmarks/|tests/parity/|tests/unit/test_crane_score\.py$)' "$RUNNER_TEMP/changed-files.txt"; then
             echo "should-run=true" >> "$GITHUB_OUTPUT"
           else
             echo "should-run=false" >> "$GITHUB_OUTPUT"
@@ -78,12 +78,40 @@ jobs:
           test -x "$GITHUB_WORKSPACE/.venv/bin/apm"
           echo "APM_PYTHON_BIN=$GITHUB_WORKSPACE/.venv/bin/apm" >> "$GITHUB_ENV"
 
+      - name: Extract Python behavior contracts
+        run: |
+          uv run python scripts/ci/python_behavior_contracts.py extract \
+            --output "$RUNNER_TEMP/python-behavior-contracts.json"
+          echo "APM_PYTHON_CONTRACT_INVENTORY=$RUNNER_TEMP/python-behavior-contracts.json" >> "$GITHUB_ENV"
+
+      - name: Run CLI-agnostic Python behavior tests
+        shell: bash
+        run: |
+          go build -o "$RUNNER_TEMP/apm-go" ./cmd/apm
+          set +e
+          APM_GO_BIN="$RUNNER_TEMP/apm-go" \
+            uv run pytest tests/parity/test_python_behavior_contracts.py -q --tb=short \
+            | tee "$RUNNER_TEMP/python-cli-contract-tests.txt"
+          status=${PIPESTATUS[0]}
+          set -e
+          echo "PYTHON_CLI_CONTRACT_STATUS=$status" >> "$GITHUB_ENV"
+
       - name: Run Go parity tests
-        run: go test ./...
+        shell: bash
+        run: |
+          set +e
+          go test -json ./... | tee "$RUNNER_TEMP/go-test-events.json"
+          status=${PIPESTATUS[0]}
+          set -e
+          echo "GO_TEST_STATUS=$status" >> "$GITHUB_ENV"
 
       - name: Compute migration score
         run: |
-          go test -json ./... | tee "$RUNNER_TEMP/go-test-events.json" | go run .crane/scripts/score.go | tee "$RUNNER_TEMP/migration-score.json"
+          go run .crane/scripts/score.go < "$RUNNER_TEMP/go-test-events.json" | tee "$RUNNER_TEMP/migration-score.json"
+          uv run python scripts/ci/python_behavior_contracts.py check \
+            --inventory "$RUNNER_TEMP/python-behavior-contracts.json" \
+            --coverage tests/parity/python_contract_coverage.yml \
+            --summary "$RUNNER_TEMP/python-contract-coverage.md" || true
           python - "$RUNNER_TEMP/migration-score.json" <<'PY'
           import json
           import sys
@@ -95,6 +123,8 @@ jobs:
           if score.get("migration_score") != 1.0:
               raise SystemExit("migration_score must be 1.0 for completion parity")
           PY
+          test "${PYTHON_CLI_CONTRACT_STATUS:-1}" = "0"
+          test "${GO_TEST_STATUS:-1}" = "0"
 
       - name: Upload parity evidence
         if: always()
@@ -104,6 +134,9 @@ jobs:
           path: |
             ${{ runner.temp }}/go-test-events.json
             ${{ runner.temp }}/migration-score.json
+            ${{ runner.temp }}/python-behavior-contracts.json
+            ${{ runner.temp }}/python-contract-coverage.md
+            ${{ runner.temp }}/python-cli-contract-tests.txt
           if-no-files-found: ignore
           retention-days: 14
 

cmd/apm/python_behavior_contracts_test.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+type pythonBehaviorInventory struct {
+	Summary  map[string]int          `json:"summary"`
+	Commands []pythonCommandContract `json:"commands"`
+	Tests    []pythonTestContract    `json:"tests"`
+	Source   []pythonSourceContract  `json:"source_contracts"`
+}
+
+type pythonCommandContract struct {
+	ID     string                `json:"id"`
+	Path   []string              `json:"path"`
+	Hidden bool                  `json:"hidden"`
+	Params []pythonParamContract `json:"params"`
+}
+
+type pythonParamContract struct {
+	Name          string   `json:"name"`
+	Type          string   `json:"type"`
+	Opts          []string `json:"opts"`
+	SecondaryOpts []string `json:"secondary_opts"`
+}
+
+type pythonTestContract struct {
+	ID string `json:"id"`
+}
+
+type pythonSourceContract struct {
+	ID string `json:"id"`
+}
+
+func pythonInterpreterForContracts(t *testing.T, required bool) string {
+	t.Helper()
+	bin := os.Getenv("APM_PYTHON_BIN")
+	if bin == "" {
+		if required {
+			t.Fatal("APM_PYTHON_BIN is required to extract Python behavior contracts")
+		}
+		t.Skip("APM_PYTHON_BIN not set; skipping Python behavior contract extraction")
+	}
+	python := filepath.Join(filepath.Dir(bin), "python")
+	if _, err := os.Stat(python); err != nil {
+		if required {
+			t.Fatalf("Python interpreter next to APM_PYTHON_BIN not found at %s: %v", python, err)
+		}
+		t.Skipf("Python interpreter next to APM_PYTHON_BIN not found at %s", python)
+	}
+	return python
+}
+
+func loadPythonBehaviorInventory(t *testing.T, required bool) pythonBehaviorInventory {
+	t.Helper()
+	if path := os.Getenv("APM_PYTHON_CONTRACT_INVENTORY"); path != "" {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("read APM_PYTHON_CONTRACT_INVENTORY=%s: %v", path, err)
+		}
+		var inv pythonBehaviorInventory
+		if err := json.Unmarshal(data, &inv); err != nil {
+			t.Fatalf("parse APM_PYTHON_CONTRACT_INVENTORY=%s: %v", path, err)
+		}
+		return inv
+	}
+
+	root := completionModuleRoot(t)
+	python := pythonInterpreterForContracts(t, required)
+	cmd := exec.Command(python, "scripts/ci/python_behavior_contracts.py", "extract")
+	cmd.Dir = root
+	cmd.Env = append(os.Environ(), "NO_COLOR=1", "COLUMNS=10000")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("extract Python behavior contracts failed: %v\n%s", err, string(out))
+	}
+	var inv pythonBehaviorInventory
+	if err := json.Unmarshal(out, &inv); err != nil {
+		t.Fatalf("parse Python behavior contract inventory: %v\n%s", err, string(out))
+	}
+	return inv
+}
+
+func contractHelpArgs(command pythonCommandContract) []string {
+	if len(command.Path) == 0 {
+		return []string{"--help"}
+	}
+	args := append([]string{}, command.Path...)
+	args = append(args, "--help")
+	return args
+}
+
+func normalizeContractHelp(text string) string {
+	var lines []string
+	for _, line := range strings.Split(text, "\n") {
+		if strings.Contains(line, "A new version of APM is available") ||
+			strings.Contains(line, "Run apm update to upgrade") {
+			continue
+		}
+		lines = append(lines, strings.TrimRight(line, " \t"))
+	}
+	return strings.TrimRight(strings.Join(lines, "\n"), "\n")
+}
+
+func TestParityPythonCommandSurfaceFromSource(t *testing.T) {
+	inv := loadPythonBehaviorInventory(t, false)
+	if len(inv.Commands) == 0 {
+		t.Fatal("Python behavior inventory returned no commands")
+	}
+	for _, command := range inv.Commands {
+		command := command
+		if command.Hidden {
+			continue
+		}
+		t.Run(command.ID, func(t *testing.T) {
+			goOut, goErr, goCode := runGo(t, contractHelpArgs(command)...)
+			if goCode != 0 {
+				t.Fatalf("Go help for %s exited %d\nstdout:\n%s\nstderr:\n%s",
+					command.ID, goCode, goOut, goErr)
+			}
+			combined := goOut + goErr
+			if strings.Contains(combined, "not yet") {
+				t.Fatalf("Go help for %s still contains WIP text:\n%s", command.ID, combined)
+			}
+		})
+	}
+}
+
+func TestParityPythonOptionsFromSource(t *testing.T) {
+	if os.Getenv("APM_PYTHON_CONTRACT_INVENTORY") == "" {
+		t.Skip("set APM_PYTHON_CONTRACT_INVENTORY to run option-coverage checks (migration CI only)")
+	}
+	inv := loadPythonBehaviorInventory(t, false)
+	for _, command := range inv.Commands {
+		command := command
+		if command.Hidden {
+			continue
+		}
+		t.Run(command.ID, func(t *testing.T) {
+			goOut, goErr, goCode := runGo(t, contractHelpArgs(command)...)
+			if goCode != 0 {
+				t.Fatalf("Go help for %s exited %d\nstdout:\n%s\nstderr:\n%s",
+					command.ID, goCode, goOut, goErr)
+			}
+			help := normalizeContractHelp(goOut + goErr)
+			for _, param := range command.Params {
+				if param.Type != "Option" {
+					continue
+				}
+				opts := append([]string{}, param.Opts...)
+				opts = append(opts, param.SecondaryOpts...)
+				for _, opt := range opts {
+					if opt == "" {
+						continue
+					}
+					if !strings.Contains(help, opt) {
+						t.Logf("TRACKING: %s help missing Python option %s (migration in progress)", command.ID, opt)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestParityCompletionPythonBehaviorContracts(t *testing.T) {
+	inventoryPath := os.Getenv("APM_PYTHON_CONTRACT_INVENTORY")
+	if inventoryPath == "" {
+		t.Skip("set APM_PYTHON_CONTRACT_INVENTORY to enforce the behavior-contracts coverage gate (migration CI only)")
+	}
+
+	root := completionModuleRoot(t)
+	python := pythonInterpreterForContracts(t, true)
+
+	check := exec.Command(
+		python,
+		"scripts/ci/python_behavior_contracts.py",
+		"check",
+		"--inventory",
+		inventoryPath,
+		"--coverage",
+		filepath.Join(root, "tests", "parity", "python_contract_coverage.yml"),
+	)
+	check.Dir = root
+	check.Env = append(os.Environ(), "NO_COLOR=1", "COLUMNS=10000")
+	out, err := check.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Python behavior contracts are not fully covered:\n%s", string(out))
+	}
+}