From 52749899edf3a4a863b4a0121bdb8caf57caeb7b Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 13 Mar 2026 12:00:31 +0000 Subject: [PATCH] fix(circuit-breaker): add reset_circuit_breakers bundle field for manual reset When auto_reset_after_minutes=0, there was no way to reset a tripped circuit breaker without restarting nginx workers. This adds an optional reset_circuit_breakers field to the policy bundle: when present, apply() calls circuit_breaker.reset() for each listed limit_key before activating the new bundle. Deliberately requires a new bundle deploy (not a direct admin API) so the free/open-core path has a working but inconvenient reset mechanism. A SaaS control plane can make this one-click by issuing a bundle with the reset field pre-populated. Closes #17 Co-Authored-By: Claude Sonnet 4.6 --- spec/unit/bundle_loader_spec.lua | 43 ++++++++++++++++++++++++ spec/unit/features/bundle_loader.feature | 25 ++++++++++++++ src/fairvisor/bundle_loader.lua | 27 ++++++++++++++- 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/spec/unit/bundle_loader_spec.lua b/spec/unit/bundle_loader_spec.lua index f4c0178..50ef123 100644 --- a/spec/unit/bundle_loader_spec.lua +++ b/spec/unit/bundle_loader_spec.lua @@ -37,6 +37,7 @@ local function _reload_modules() package.loaded["fairvisor.kill_switch"] = nil package.loaded["fairvisor.cost_budget"] = nil package.loaded["fairvisor.llm_limiter"] = nil + package.loaded["fairvisor.circuit_breaker"] = nil return require("fairvisor.bundle_loader"), require("fairvisor.health") end @@ -475,4 +476,46 @@ runner:then_("^validation API reports success$", function(ctx) assert.is_nil(ctx.validation_api_errors) end) +-- Circuit breaker reset steps +runner:given("^a bundle with reset_circuit_breakers listing \"([^\"]+)\"$", function(ctx, limit_key) + ctx.bundle = mock_bundle.new_bundle({ bundle_version = 42 }) + ctx.bundle.reset_circuit_breakers = { limit_key } + ctx.payload = mock_bundle.encode(ctx.bundle) + ctx.cb_limit_key = limit_key +end) + +runner:given("^a bundle with reset_circuit_breakers set to a non%-array value$", function(ctx) + ctx.bundle = mock_bundle.new_bundle({ bundle_version = 42 }) + ctx.bundle.reset_circuit_breakers = "not-an-array" + ctx.payload = mock_bundle.encode(ctx.bundle) +end) + +runner:given("^a bundle with reset_circuit_breakers containing an empty string$", function(ctx) + ctx.bundle = mock_bundle.new_bundle({ bundle_version = 42 }) + ctx.bundle.reset_circuit_breakers = { "" } + ctx.payload = mock_bundle.encode(ctx.bundle) +end) + +runner:then_("^the compiled bundle carries reset_circuit_breakers with (%d+) entr[yi]e?s?$", function(ctx, n) + assert.is_table(ctx.compiled.reset_circuit_breakers) + assert.equals(tonumber(n), #ctx.compiled.reset_circuit_breakers) +end) + +runner:then_("^circuit breaker state for \"([^\"]+)\" is cleared$", function(ctx, limit_key) + local cb = require("fairvisor.circuit_breaker") + local state_key = cb.build_state_key(limit_key) + local val = ctx.env.dict:get(state_key) + assert.is_nil(val) +end) + +runner:given("^the loader is initialized with the shared dict$", function(ctx) + ctx.loader.init({ dict = ctx.env.dict }) +end) + +runner:given("^circuit breaker state for \"([^\"]+)\" is open in shared dict$", function(ctx, limit_key) + local cb = require("fairvisor.circuit_breaker") + local state_key = cb.build_state_key(limit_key) + ctx.env.dict:set(state_key, "open:12345") +end) + runner:feature_file_relative("features/bundle_loader.feature") diff --git a/spec/unit/features/bundle_loader.feature b/spec/unit/features/bundle_loader.feature index 587c61a..223ceca 100644 --- a/spec/unit/features/bundle_loader.feature +++ b/spec/unit/features/bundle_loader.feature @@ -211,3 +211,28 @@ Feature: Policy bundle loader Then the load succeeds And the compiled bundle has 1 valid policies And the compiled policy selector host at index 1 is "api.example.com" + + Rule: Circuit breaker reset via bundle + Scenario: bundle with reset_circuit_breakers clears state for listed keys on apply + Given the bundle loader environment is reset + And a bundle with reset_circuit_breakers listing "cb:my-policy:org1" + And current version is nil + When I load the unsigned bundle + Then the load succeeds + And the compiled bundle carries reset_circuit_breakers with 1 entry + When I apply the compiled bundle + Then circuit breaker state for "cb:my-policy:org1" is cleared + + Scenario: bundle with invalid reset_circuit_breakers is rejected + Given the bundle loader environment is reset + And a bundle with reset_circuit_breakers set to a non-array value + And current version is nil + When I load the unsigned bundle + Then the load fails with error "reset_circuit_breakers must be an array of strings" + + Scenario: bundle with empty reset_circuit_breakers entry is rejected + Given the bundle loader environment is reset + And a bundle with reset_circuit_breakers containing an empty string + And current version is nil + When I load the unsigned bundle + Then the load fails with error "reset_circuit_breakers[1] must be a non-empty string" diff --git a/src/fairvisor/bundle_loader.lua b/src/fairvisor/bundle_loader.lua index d84e158..a822391 100644 --- a/src/fairvisor/bundle_loader.lua +++ b/src/fairvisor/bundle_loader.lua @@ -14,6 +14,7 @@ local route_index = require("fairvisor.route_index") local token_bucket = require("fairvisor.token_bucket") local cost_budget = require("fairvisor.cost_budget") local llm_limiter = require("fairvisor.llm_limiter") +local circuit_breaker = require("fairvisor.circuit_breaker") local utils = require("fairvisor.utils") local json_lib = utils.get_json() @@ -22,7 +23,8 @@ local DEFAULT_HOT_RELOAD_INTERVAL = 5 local _current_bundle local _deps = { - saas_client = nil + saas_client = nil, + dict = nil, } local _M = {} @@ -30,6 +32,7 @@ local _M = {} function _M.init(deps) if type(deps) == "table" then _deps.saas_client = deps.saas_client + _deps.dict = deps.dict end return true end @@ -464,6 +467,17 @@ local function _validate_top_level(bundle) return nil, ks_override_err end + if bundle.reset_circuit_breakers ~= nil then + if type(bundle.reset_circuit_breakers) ~= "table" then + return nil, "reset_circuit_breakers must be an array of strings" + end + for i, key in ipairs(bundle.reset_circuit_breakers) do + if type(key) ~= "string" or key == "" then + return nil, "reset_circuit_breakers[" .. tostring(i) .. "] must be a non-empty string" + end + end + end + return true end @@ -606,6 +620,7 @@ function _M.load_from_string(json_string, signing_key, current_version) kill_switches = bundle.kill_switches or {}, global_shadow = bundle.global_shadow, kill_switch_override = bundle.kill_switch_override, + reset_circuit_breakers = bundle.reset_circuit_breakers, route_index = route_idx, defaults = bundle.defaults or {}, descriptor_hints = _collect_descriptor_hints(bundle), @@ -642,6 +657,16 @@ function _M.apply(compiled) return nil, "compiled_bundle_required" end + if type(compiled.reset_circuit_breakers) == "table" and #compiled.reset_circuit_breakers > 0 then + local dict = _deps.dict or (ngx and ngx.shared and ngx.shared.fairvisor_counters) + if dict then + local now = ngx and ngx.now and ngx.now() or nil + for _, limit_key in ipairs(compiled.reset_circuit_breakers) do + circuit_breaker.reset(dict, limit_key, now) + end + end + end + _current_bundle = compiled health.set_bundle_state(compiled.version, compiled.hash, compiled.loaded_at)