diff --git a/README.md b/README.md index 5dddd737..4ee61c90 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,9 @@ OPTIMIZATIONS: -delay value duration between each http request (eg: 200ms, 1s) (default -1ns) -rsts, -response-size-to-save int max response size to save in bytes (default 512000000) -rstr, -response-size-to-read int max response size to read in bytes (default 512000000) + -retry-rounds int number of retry rounds for HTTP 429 responses (Too Many Requests) + -retry-delay int fallback delay in ms when Retry-After header is absent (HTTP 429) (default 500) + -retry-timeout int max total time in seconds for retry rounds (HTTP 429) (default 30) CLOUD: -auth configure projectdiscovery cloud (pdcp) api key (default true) @@ -310,6 +313,7 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http username: admin password: secret ``` +- The `-retry-rounds` flag enables automatic retries for HTTP 429 (Too Many Requests) responses. When a server returns a `Retry-After` header, httpx respects it to determine the wait time before retrying. If the header is absent, the `-retry-delay` value is used as a fallback. The `-retry-timeout` flag sets a hard upper bound on total retry time to prevent indefinite blocking. - The following flags should be used for specific use cases instead of running them as default with other probes: - `-ports` - `-path` diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 10179973..85b6d65a 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "strings" + "sync/atomic" "github.com/julienschmidt/httprouter" "github.com/projectdiscovery/httpx/internal/testutils" @@ -33,6 +34,9 @@ var httpTestcases = map[string]testutils.TestCase{ "Output Match Condition": &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}}, "Output Filter Condition": &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}}, "Output All": &outputAll{}, + "Retry 429 with Retry-After header": &retry429WithHeader{}, + "Retry 429 with fallback delay": &retry429FallbackDelay{}, + "Retry 429 respects timeout": &retry429Timeout{}, } type standardHttpGet struct { @@ -419,3 +423,119 @@ func (h *outputAll) Execute() error { return nil } + +type retry429WithHeader struct{} + +func (h *retry429WithHeader) Execute() error { + var hits int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&hits, 1) + if n < 3 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, "OK") + })) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL, debug, + "-retry-rounds", "3", + "-retry-delay", "100", + "-retry-timeout", "15", + "-status-code", "-no-color", + ) + if err != nil { + return err + } + + var has200 bool + for _, line := range results { + if strings.Contains(line, "[200]") { + has200 = true + } + } + if !has200 { + return fmt.Errorf("expected 200 after retrying 429, got results: %v", results) + } + + totalHits := atomic.LoadInt32(&hits) + if totalHits < 3 { + return fmt.Errorf("expected at least 3 server hits (429->429->200), got %d", totalHits) + } + return nil +} + +type retry429FallbackDelay struct{} + +func (h *retry429FallbackDelay) Execute() error { + var hits int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&hits, 1) + if n < 2 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, "OK") + })) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL, debug, + "-retry-rounds", "3", + "-retry-delay", "200", + "-retry-timeout", "15", + "-status-code", "-no-color", + ) + if err != nil { + return err + } + + var has200 bool + for _, line := range results { + if strings.Contains(line, "[200]") { + has200 = true + } + } + if !has200 { + return fmt.Errorf("expected 200 after retrying 429 with fallback delay, got results: %v", results) + } + + totalHits := atomic.LoadInt32(&hits) + if totalHits < 2 { + return fmt.Errorf("expected at least 2 server hits (429->200), got %d", totalHits) + } + return nil +} + +type retry429Timeout struct{} + +func (h *retry429Timeout) Execute() error { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "60") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL, debug, + "-retry-rounds", "5", + "-retry-delay", "60000", + "-retry-timeout", "3", + "-status-code", "-no-color", + ) + if err != nil { + return err + } + + var has429 bool + for _, line := range results { + if strings.Contains(line, "[429]") { + has429 = true + } + } + if !has429 { + return fmt.Errorf("expected 429 in output, got results: %v", results) + } + return nil +} diff --git a/runner/options.go b/runner/options.go index d12a7dad..ea1424a3 100644 --- a/runner/options.go +++ b/runner/options.go @@ -189,20 +189,20 @@ type Options struct { // Deprecated: use Proxy HTTPProxy string // Deprecated: use Proxy - SocksProxy string - Proxy string - InputFile string - InputMode string - InputTargetHost goflags.StringSlice - Methods string - RequestURI string - RequestURIs string - requestURIs []string - OutputMatchStatusCode string - OutputMatchContentLength string - OutputFilterStatusCode string + SocksProxy string + Proxy string + InputFile string + InputMode string + InputTargetHost goflags.StringSlice + Methods string + RequestURI string + RequestURIs string + requestURIs []string + OutputMatchStatusCode string + OutputMatchContentLength string + OutputFilterStatusCode string // Deprecated: use OutputFilterPageType with "error" instead. - OutputFilterErrorPage bool + OutputFilterErrorPage bool OutputFilterPageType goflags.StringSlice FilterOutDuplicates bool OutputFilterContentLength string @@ -287,6 +287,9 @@ type Options struct { RateLimitMinute int Probe bool Resume bool + RetryRounds int + RetryDelay int + RetryTimeout int resumeCfg *ResumeCfg Exclude goflags.StringSlice HostMaxErrors int @@ -578,6 +581,9 @@ func ParseOptions() *Options { flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to save in bytes"), flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to read in bytes"), + flagSet.IntVar(&options.RetryRounds, "retry-rounds", 0, "number of retry rounds for HTTP 429 responses (Too Many Requests)"), + flagSet.IntVar(&options.RetryDelay, "retry-delay", 500, "fallback delay in ms when Retry-After header is absent (HTTP 429)"), + flagSet.IntVar(&options.RetryTimeout, "retry-timeout", 30, "max total time in seconds for retry rounds (HTTP 429)"), ) flagSet.CreateGroup("cloud", "Cloud", @@ -840,6 +846,15 @@ func (options *Options) ValidateOptions() error { options.Threads = defaultThreads } + if options.RetryRounds > 0 { + if options.RetryDelay <= 0 { + return errors.New(fmt.Sprintf("invalid retry-delay: must be >0 when retry-rounds=%d (got %d)", options.RetryRounds, options.RetryDelay)) + } + if options.RetryTimeout <= 0 { + return errors.New(fmt.Sprintf("invalid retry-timeout: must be >0 when retry-rounds=%d (got %d)", options.RetryRounds, options.RetryTimeout)) + } + } + return nil } diff --git a/runner/runner.go b/runner/runner.go index 6043b5f5..f5f23503 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1501,6 +1501,7 @@ func (r *Runner) RunEnumeration() { }(nextStep) wg, _ := syncutil.New(syncutil.WithSize(r.options.Threads)) + var rq retryQueue processItem := func(k string) error { select { @@ -1531,10 +1532,10 @@ func (r *Runner) RunEnumeration() { for _, p := range r.options.requestURIs { scanopts := r.scanopts.Clone() scanopts.RequestURI = p - r.process(k, wg, r.hp, protocol, scanopts, output) + r.process(k, wg, r.hp, protocol, scanopts, output, &rq) } } else { - r.process(k, wg, r.hp, protocol, &r.scanopts, output) + r.process(k, wg, r.hp, protocol, &r.scanopts, output, &rq) } } } @@ -1567,8 +1568,17 @@ func (r *Runner) RunEnumeration() { wg.Wait() - close(output) + if r.options.RetryRounds > 0 { + retryCtx := context.Background() + if r.options.RetryTimeout > 0 { + var cancel context.CancelFunc + retryCtx, cancel = context.WithTimeout(retryCtx, time.Duration(r.options.RetryTimeout)*time.Second) + defer cancel() + } + r.processRetries(retryCtx, &rq, output) + } + close(output) wgoutput.Wait() if r.scanopts.StoreVisionReconClusters { @@ -1592,6 +1602,78 @@ func (r *Runner) RunEnumeration() { } } +func retryDelay(res Result, fallbackMs int) time.Duration { + if res.Response != nil { + if ra, ok := res.Response.Headers["Retry-After"]; ok && len(ra) > 0 { + if seconds, err := strconv.Atoi(ra[0]); err == nil && seconds > 0 { + return time.Duration(seconds) * time.Second + } + if t, err := http.ParseTime(ra[0]); err == nil { + if d := time.Until(t); d > 0 { + return d + } + } + } + } + return time.Duration(fallbackMs) * time.Millisecond +} + +type retryItem struct { + hp *httpx.HTTPX + protocol string + target httpx.Target + method string + input string + scanopts *ScanOptions + delay time.Duration +} + +type retryQueue struct { + mu sync.Mutex + items []retryItem +} + +func (q *retryQueue) push(item retryItem) { + q.mu.Lock() + q.items = append(q.items, item) + q.mu.Unlock() +} + +func (q *retryQueue) drain() []retryItem { + q.mu.Lock() + items := q.items + q.items = nil + q.mu.Unlock() + return items +} + +func (r *Runner) processRetries(ctx context.Context, rq *retryQueue, output chan Result) { + for round := 0; round < r.options.RetryRounds; round++ { + items := rq.drain() + if len(items) == 0 { + return + } + for _, item := range items { + timer := time.NewTimer(item.delay) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + result := r.analyze(item.hp, item.protocol, item.target, item.method, item.input, item.scanopts) + output <- result + if result.StatusCode == http.StatusTooManyRequests { + rq.push(retryItem{ + hp: item.hp, protocol: item.protocol, target: item.target, + method: item.method, input: item.input, scanopts: item.scanopts, + delay: retryDelay(result, r.options.RetryDelay), + }) + } + } + } +} + func handleStripAnsiCharacters(data string, skip bool) string { if skip { return data @@ -1658,11 +1740,11 @@ func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts } -func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) { - r.process(t, wg, r.hp, protocol, scanopts, output) +func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result, rq *retryQueue) { + r.process(t, wg, r.hp, protocol, scanopts, output, rq) } -func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { +func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result, rq *retryQueue) { // attempts to set the workpool size to the number of threads if r.options.Threads > 0 && wg.Size != r.options.Threads { if err := wg.Resize(context.Background(), r.options.Threads); err != nil { @@ -1685,31 +1767,38 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT wg.Add() go func(target httpx.Target, method, protocol string) { defer wg.Done() - result := r.analyze(hp, protocol, target, method, t, scanopts) - output <- result - if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) + result := r.analyze(hp, protocol, target, method, t, scanopts) + output <- result + if result.StatusCode == http.StatusTooManyRequests && r.options.RetryRounds > 0 && rq != nil { + rq.push(retryItem{ + hp: hp, protocol: protocol, target: target, + method: method, input: t, scanopts: scanopts.Clone(), + delay: retryDelay(result, r.options.RetryDelay), + }) + } + if scanopts.TLSProbe && result.TLSData != nil { + for _, tt := range result.TLSData.SubjectAN { + if !r.testAndSet(tt) { + continue } + r.process(tt, wg, hp, protocol, scanopts, output, rq) + } + if r.testAndSet(result.TLSData.SubjectCN) { + r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output, rq) } - if scanopts.CSPProbe && result.CSPData != nil { - scanopts.CSPProbe = false - domains := result.CSPData.Domains - domains = append(domains, result.CSPData.Fqdns...) - for _, tt := range domains { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) + } + if scanopts.CSPProbe && result.CSPData != nil { + scanopts.CSPProbe = false + domains := result.CSPData.Domains + domains = append(domains, result.CSPData.Fqdns...) + for _, tt := range domains { + if !r.testAndSet(tt) { + continue } + r.process(tt, wg, hp, protocol, scanopts, output, rq) } - }(target, method, prot) + } + }(target, method, prot) } } } @@ -1739,20 +1828,27 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT urlx.UpdatePort(fmt.Sprint(port)) target.Host = urlx.String() } - result := r.analyze(hp, protocol, target, method, t, scanopts) - output <- result - if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) + result := r.analyze(hp, protocol, target, method, t, scanopts) + output <- result + if result.StatusCode == http.StatusTooManyRequests && r.options.RetryRounds > 0 && rq != nil { + rq.push(retryItem{ + hp: hp, protocol: protocol, target: target, + method: method, input: t, scanopts: scanopts.Clone(), + delay: retryDelay(result, r.options.RetryDelay), + }) + } + if scanopts.TLSProbe && result.TLSData != nil { + for _, tt := range result.TLSData.SubjectAN { + if !r.testAndSet(tt) { + continue } + r.process(tt, wg, hp, protocol, scanopts, output, rq) } - }(port, target, method, wantedProtocol) + if r.testAndSet(result.TLSData.SubjectCN) { + r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output, rq) + } + } + }(port, target, method, wantedProtocol) } } } diff --git a/runner/runner_test.go b/runner/runner_test.go index 832c4e36..8bf6e4c1 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1,9 +1,13 @@ package runner import ( + "context" "fmt" + "net/http" + "net/http/httptest" "os" "strings" + "sync/atomic" "testing" "time" @@ -12,6 +16,7 @@ import ( "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/mapcidr/asn" stringsutil "github.com/projectdiscovery/utils/strings" + syncutil "github.com/projectdiscovery/utils/sync" "github.com/stretchr/testify/require" ) @@ -413,3 +418,283 @@ func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) { }) } } + +func TestRunner_Process_And_RetryLoop(t *testing.T) { + var hits1, hits2 int32 + + // srv1: returns 429 for the first 3 requests, and 200 on the 4th request + srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt32(&hits1, 1) != 4 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv1.Close() + + // srv2: returns 429 for the first 2 requests, and 200 on the 3rd request + srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt32(&hits2, 1) != 3 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv2.Close() + + r, err := New(&Options{ + Threads: 1, + RetryRounds: 2, + RetryDelay: 5, + RetryTimeout: 30, + Timeout: 3, + }) + require.NoError(t, err) + defer r.Close() + + output := make(chan Result, 20) + var rq retryQueue + + wg, _ := syncutil.New(syncutil.WithSize(r.options.Threads)) + so := r.scanopts.Clone() + so.Methods = []string{"GET"} + so.TLSProbe = false + so.CSPProbe = false + + seed := map[string]string{ + "srv1": srv1.URL, + "srv2": srv2.URL, + } + + for _, url := range seed { + r.process(url, wg, r.hp, httpx.HTTP, so, output, &rq) + } + + wg.Wait() + r.processRetries(context.Background(), &rq, output) + close(output) + + var s1n429, s1n200, s2n429, s2n200 int + for res := range output { + switch res.StatusCode { + case http.StatusTooManyRequests: + if res.URL == srv1.URL { + s1n429++ + } else { + s2n429++ + } + case http.StatusOK: + if res.URL == srv1.URL { + s1n200++ + } else { + s2n200++ + } + } + } + + require.Equal(t, 3, s1n429) + require.Equal(t, 0, s1n200) + + require.Equal(t, 2, s2n429) + require.Equal(t, 1, s2n200) +} + +func TestRetryDelay_RetryAfterHeader(t *testing.T) { + fallbackMs := 500 + + t.Run("uses Retry-After seconds header", func(t *testing.T) { + res := Result{ + Response: &httpx.Response{ + Headers: map[string][]string{ + "Retry-After": {"3"}, + }, + }, + } + d := retryDelay(res, fallbackMs) + require.Equal(t, 3*time.Second, d) + }) + + t.Run("uses Retry-After HTTP-date header", func(t *testing.T) { + future := time.Now().Add(10 * time.Second) + res := Result{ + Response: &httpx.Response{ + Headers: map[string][]string{ + "Retry-After": {future.UTC().Format(http.TimeFormat)}, + }, + }, + } + d := retryDelay(res, fallbackMs) + require.InDelta(t, 10*time.Second, d, float64(2*time.Second)) + }) + + t.Run("falls back when no header", func(t *testing.T) { + res := Result{ + Response: &httpx.Response{ + Headers: map[string][]string{}, + }, + } + d := retryDelay(res, fallbackMs) + require.Equal(t, 500*time.Millisecond, d) + }) + + t.Run("falls back when response is nil", func(t *testing.T) { + res := Result{} + d := retryDelay(res, fallbackMs) + require.Equal(t, 500*time.Millisecond, d) + }) + + t.Run("falls back when Retry-After is unparseable", func(t *testing.T) { + res := Result{ + Response: &httpx.Response{ + Headers: map[string][]string{ + "Retry-After": {"not-a-number"}, + }, + }, + } + d := retryDelay(res, fallbackMs) + require.Equal(t, 500*time.Millisecond, d) + }) +} + +func TestRetryRespectsRetryAfterHeader(t *testing.T) { + var hits int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(&hits, 1) + if n < 3 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + rn, err := New(&Options{ + Threads: 1, + RetryRounds: 3, + RetryDelay: 5, + RetryTimeout: 30, + Timeout: 3, + }) + require.NoError(t, err) + defer rn.Close() + + output := make(chan Result, 10) + var rq retryQueue + + wg, _ := syncutil.New(syncutil.WithSize(1)) + so := rn.scanopts.Clone() + so.Methods = []string{"GET"} + so.TLSProbe = false + so.CSPProbe = false + + rn.process(srv.URL, wg, rn.hp, httpx.HTTP, so, output, &rq) + wg.Wait() + + start := time.Now() + rn.processRetries(context.Background(), &rq, output) + elapsed := time.Since(start) + close(output) + + var n429, n200 int + for res := range output { + switch res.StatusCode { + case http.StatusTooManyRequests: + n429++ + case http.StatusOK: + n200++ + } + } + require.Equal(t, 2, n429) + require.Equal(t, 1, n200) + require.GreaterOrEqual(t, elapsed, 2*time.Second, "should have waited ~1s per Retry-After") +} + +func TestRetryRespectsTimeout(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "60") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + rn, err := New(&Options{ + Threads: 1, + RetryRounds: 10, + RetryDelay: 60000, + RetryTimeout: 2, + Timeout: 3, + }) + require.NoError(t, err) + defer rn.Close() + + output := make(chan Result, 100) + var rq retryQueue + + wg, _ := syncutil.New(syncutil.WithSize(1)) + so := rn.scanopts.Clone() + so.Methods = []string{"GET"} + so.TLSProbe = false + so.CSPProbe = false + + rn.process(srv.URL, wg, rn.hp, httpx.HTTP, so, output, &rq) + wg.Wait() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + start := time.Now() + rn.processRetries(ctx, &rq, output) + elapsed := time.Since(start) + close(output) + + require.Less(t, elapsed, 10*time.Second, "retry should have been cancelled by context timeout") + + var n429 int + for res := range output { + if res.StatusCode == http.StatusTooManyRequests { + n429++ + } + } + require.GreaterOrEqual(t, n429, 1, "should have received at least the initial 429") +} + +func TestRetryNo429_CompletesNormally(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + rn, err := New(&Options{ + Threads: 1, + RetryRounds: 3, + RetryDelay: 500, + RetryTimeout: 30, + Timeout: 3, + }) + require.NoError(t, err) + defer rn.Close() + + output := make(chan Result, 10) + var rq retryQueue + + wg, _ := syncutil.New(syncutil.WithSize(1)) + so := rn.scanopts.Clone() + so.Methods = []string{"GET"} + so.TLSProbe = false + so.CSPProbe = false + + rn.process(srv.URL, wg, rn.hp, httpx.HTTP, so, output, &rq) + wg.Wait() + rn.processRetries(context.Background(), &rq, output) + close(output) + + var n200 int + for res := range output { + if res.StatusCode == http.StatusOK { + n200++ + } + } + require.Equal(t, 1, n200) + require.Empty(t, rq.items, "no retries should have been queued") +}