From 934da3eb2c70470f2125111a3fe8bee64ccf40c7 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Thu, 7 May 2026 21:18:07 +0800 Subject: [PATCH 1/2] fix(test): make no-provider tests robust against real ~/.caseforge.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestAskCommand_FailsWhenNoProvider and TestAskCommand_Integration_NoProvider were relying on env-var clearing, which doesn't work when ~/.caseforge.yaml has a real api_key (config file takes precedence over empty env vars in viper). - FailsWhenNoProvider: reset viper and set ai.provider=noop directly - Integration_NoProvider: pass --config pointing to a temp noop-provider yaml so initConfig() reads that instead of ~/.caseforge.yaml AT-235: same root cause — conformance --config forces the "LLM provider not available" path regardless of the local config file. --- cmd/ask_test.go | 29 ++++++++++++++++++----------- scripts/acceptance.sh | 4 +++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/ask_test.go b/cmd/ask_test.go index b606482..5d465cf 100644 --- a/cmd/ask_test.go +++ b/cmd/ask_test.go @@ -1,8 +1,11 @@ package cmd import ( + "os" + "path/filepath" "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -40,27 +43,31 @@ func TestAskCommand_RequiresDescription(t *testing.T) { } func TestAskCommand_FailsWhenNoProvider(t *testing.T) { - // Clear all API key env vars so config.Load() returns a noop provider. - t.Setenv("ANTHROPIC_API_KEY", "") - t.Setenv("OPENAI_API_KEY", "") - t.Setenv("GEMINI_API_KEY", "") - // NoopProvider.IsAvailable() returns false → Generator.Generate returns error. + // Reset viper and set noop provider — prior tests may have loaded ~/.caseforge.yaml + // via rootCmd.Execute(), leaving a real API key in viper's state. + viper.Reset() + t.Cleanup(func() { viper.Reset() }) + viper.Set("ai.provider", "noop") + err := runAsk(askCmd, []string{"POST /users - create user"}) require.Error(t, err) assert.Contains(t, err.Error(), "AI provider") } func TestAskCommand_Integration_NoProvider(t *testing.T) { - // Clear env so no real LLM is configured. - t.Setenv("ANTHROPIC_API_KEY", "") - t.Setenv("OPENAI_API_KEY", "") - t.Setenv("GEMINI_API_KEY", "") + // Use a temp config with provider=noop so initConfig() doesn't pick up + // the real ~/.caseforge.yaml (which may have a real API key). + tmpCfg := filepath.Join(t.TempDir(), ".caseforge.yaml") + require.NoError(t, os.WriteFile(tmpCfg, []byte("ai:\n provider: noop\n"), 0600)) + + viper.Reset() + t.Cleanup(func() { viper.Reset() }) outDir := t.TempDir() t.Cleanup(func() { rootCmd.SetArgs(nil) }) - rootCmd.SetArgs([]string{"ask", "--output", outDir, "POST /users create user"}) + rootCmd.SetArgs([]string{"ask", "--config", tmpCfg, "--output", outDir, "POST /users create user"}) err := rootCmd.Execute() - // With no provider configured, we expect an "AI provider" error. + // With noop provider, Generator.Generate returns "AI provider" error. require.Error(t, err) assert.Contains(t, err.Error(), "AI provider") } diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 7b165d2..7e7b7f9 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1503,8 +1503,10 @@ contains "AT-233" "conformance --spec required" "required flag" \ "$BIN conformance --target http://localhost:8080 2>&1 || true" contains "AT-234" "conformance --target required" "required flag" \ "(cd $REPO_ROOT && $BIN conformance --spec cmd/testdata/crud.yaml 2>&1 || true)" +NOPROV_CFG=$(mktemp /tmp/caseforge-noprov-XXXXXX.yaml) +printf 'ai:\n provider: noop\n' > "$NOPROV_CFG" contains "AT-235" "conformance no LLM fails gracefully" "LLM provider not available" \ - "(cd $REPO_ROOT && $BIN conformance --spec cmd/testdata/crud.yaml --target http://localhost:8080 2>&1 || true)" + "(cd $REPO_ROOT && $BIN conformance --config '$NOPROV_CFG' --spec cmd/testdata/crud.yaml --target http://localhost:8080 2>&1 || true)" echo "" From b1204d632d166ab64526bd33103b5766b25b9536 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Thu, 7 May 2026 21:30:15 +0800 Subject: [PATCH 2/2] fix(test): reset cfgFile var and keep noprov config in WORKDIR Reviewer feedback on #71: - TestAskCommand_Integration_NoProvider was leaving cfgFile set to a deleted temp path; subsequent rootCmd.Execute() calls in the same test binary printed "error reading config: no such file or directory" to stderr. Added cfgFile = "" to the cleanup closure. - AT-235 noprov config was written to /tmp and leaked across runs. Moved to $WORKDIR so the existing trap rm cleans it up automatically. - Switched '$NOPROV_CFG' (single-quotes inside double-quotes) to \"$NOPROV_CFG\" for idiomatic quoting consistency. --- cmd/ask_test.go | 5 ++++- scripts/acceptance.sh | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/ask_test.go b/cmd/ask_test.go index 5d465cf..29b4b16 100644 --- a/cmd/ask_test.go +++ b/cmd/ask_test.go @@ -61,7 +61,10 @@ func TestAskCommand_Integration_NoProvider(t *testing.T) { require.NoError(t, os.WriteFile(tmpCfg, []byte("ai:\n provider: noop\n"), 0600)) viper.Reset() - t.Cleanup(func() { viper.Reset() }) + t.Cleanup(func() { + viper.Reset() + cfgFile = "" // prevent stale temp path from leaking into subsequent rootCmd.Execute() calls + }) outDir := t.TempDir() t.Cleanup(func() { rootCmd.SetArgs(nil) }) diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 7e7b7f9..e9801c4 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1503,10 +1503,10 @@ contains "AT-233" "conformance --spec required" "required flag" \ "$BIN conformance --target http://localhost:8080 2>&1 || true" contains "AT-234" "conformance --target required" "required flag" \ "(cd $REPO_ROOT && $BIN conformance --spec cmd/testdata/crud.yaml 2>&1 || true)" -NOPROV_CFG=$(mktemp /tmp/caseforge-noprov-XXXXXX.yaml) +NOPROV_CFG="$WORKDIR/caseforge-noprov.yaml" printf 'ai:\n provider: noop\n' > "$NOPROV_CFG" contains "AT-235" "conformance no LLM fails gracefully" "LLM provider not available" \ - "(cd $REPO_ROOT && $BIN conformance --config '$NOPROV_CFG' --spec cmd/testdata/crud.yaml --target http://localhost:8080 2>&1 || true)" + "(cd $REPO_ROOT && $BIN conformance --config \"$NOPROV_CFG\" --spec cmd/testdata/crud.yaml --target http://localhost:8080 2>&1 || true)" echo ""