feat: retry on 429

README.md

@@ -267,6 +267,9 @@ OPTIMIZATIONS:
    -delay value                       duration between each http request (eg: 200ms, 1s) (default -1ns)
    -rsts, -response-size-to-save int  max response size to save in bytes (default 512000000)
    -rstr, -response-size-to-read int  max response size to read in bytes (default 512000000)
+   -retry-rounds int                  number of retry rounds for HTTP 429 responses (Too Many Requests)
+   -retry-delay int                   fallback delay in ms when Retry-After header is absent (HTTP 429) (default 500)
+   -retry-timeout int                 max total time in seconds for retry rounds (HTTP 429) (default 30)
 
 CLOUD:
    -auth                           configure projectdiscovery cloud (pdcp) api key (default true)
@@ -310,6 +313,7 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http
       username: admin
       password: secret
   ```
+- The `-retry-rounds` flag enables automatic retries for HTTP 429 (Too Many Requests) responses. When a server returns a `Retry-After` header, httpx respects it to determine the wait time before retrying. If the header is absent, the `-retry-delay` value is used as a fallback. The `-retry-timeout` flag sets a hard upper bound on total retry time to prevent indefinite blocking.
 - The following flags should be used for specific use cases instead of running them as default with other probes:
    - `-ports`
    - `-path`

cmd/integration-test/http.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"strings"
+	"sync/atomic"
 
 	"github.com/julienschmidt/httprouter"
 	"github.com/projectdiscovery/httpx/internal/testutils"
@@ -33,6 +34,9 @@ var httpTestcases = map[string]testutils.TestCase{
 	"Output Match Condition":                                                              &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}},
 	"Output Filter Condition":                                                             &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}},
 	"Output All":                                                                          &outputAll{},
+	"Retry 429 with Retry-After header":                                                   &retry429WithHeader{},
+	"Retry 429 with fallback delay":                                                       &retry429FallbackDelay{},
+	"Retry 429 respects timeout":                                                          &retry429Timeout{},
 }
 
 type standardHttpGet struct {
@@ -419,3 +423,119 @@ func (h *outputAll) Execute() error {
 
 	return nil
 }
+
+type retry429WithHeader struct{}
+
+func (h *retry429WithHeader) Execute() error {
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&hits, 1)
+		if n < 3 {
+			w.Header().Set("Retry-After", "1")
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprint(w, "OK")
+	}))
+	defer ts.Close()
+
+	results, err := testutils.RunHttpxAndGetResults(ts.URL, debug,
+		"-retry-rounds", "3",
+		"-retry-delay", "100",
+		"-retry-timeout", "15",
+		"-status-code", "-no-color",
+	)
+	if err != nil {
+		return err
+	}
+
+	var has200 bool
+	for _, line := range results {
+		if strings.Contains(line, "[200]") {
+			has200 = true
+		}
+	}
+	if !has200 {
+		return fmt.Errorf("expected 200 after retrying 429, got results: %v", results)
+	}
+
+	totalHits := atomic.LoadInt32(&hits)
+	if totalHits < 3 {
+		return fmt.Errorf("expected at least 3 server hits (429->429->200), got %d", totalHits)
+	}
+	return nil
+}
+
+type retry429FallbackDelay struct{}
+
+func (h *retry429FallbackDelay) Execute() error {
+	var hits int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&hits, 1)
+		if n < 2 {
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		_, _ = fmt.Fprint(w, "OK")
+	}))
+	defer ts.Close()
+
+	results, err := testutils.RunHttpxAndGetResults(ts.URL, debug,
+		"-retry-rounds", "3",
+		"-retry-delay", "200",
+		"-retry-timeout", "15",
+		"-status-code", "-no-color",
+	)
+	if err != nil {
+		return err
+	}
+
+	var has200 bool
+	for _, line := range results {
+		if strings.Contains(line, "[200]") {
+			has200 = true
+		}
+	}
+	if !has200 {
+		return fmt.Errorf("expected 200 after retrying 429 with fallback delay, got results: %v", results)
+	}
+
+	totalHits := atomic.LoadInt32(&hits)
+	if totalHits < 2 {
+		return fmt.Errorf("expected at least 2 server hits (429->200), got %d", totalHits)
+	}
+	return nil
+}
+
+type retry429Timeout struct{}
+
+func (h *retry429Timeout) Execute() error {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Retry-After", "60")
+		w.WriteHeader(http.StatusTooManyRequests)
+	}))
+	defer ts.Close()
+
+	results, err := testutils.RunHttpxAndGetResults(ts.URL, debug,
+		"-retry-rounds", "5",
+		"-retry-delay", "60000",
+		"-retry-timeout", "3",
+		"-status-code", "-no-color",
+	)
+	if err != nil {
+		return err
+	}
+
+	var has429 bool
+	for _, line := range results {
+		if strings.Contains(line, "[429]") {
+			has429 = true
+		}
+	}
+	if !has429 {
+		return fmt.Errorf("expected 429 in output, got results: %v", results)
+	}
+	return nil
+}

runner/options.go

@@ -189,20 +189,20 @@ type Options struct {
 	// Deprecated: use Proxy
 	HTTPProxy string
 	// Deprecated: use Proxy
-	SocksProxy                string
-	Proxy                     string
-	InputFile                 string
-	InputMode                 string
-	InputTargetHost           goflags.StringSlice
-	Methods                   string
-	RequestURI                string
-	RequestURIs               string
-	requestURIs               []string
-	OutputMatchStatusCode     string
-	OutputMatchContentLength  string
-	OutputFilterStatusCode    string
+	SocksProxy               string
+	Proxy                    string
+	InputFile                string
+	InputMode                string
+	InputTargetHost          goflags.StringSlice
+	Methods                  string
+	RequestURI               string
+	RequestURIs              string
+	requestURIs              []string
+	OutputMatchStatusCode    string
+	OutputMatchContentLength string
+	OutputFilterStatusCode   string
 	// Deprecated: use OutputFilterPageType with "error" instead.
-	OutputFilterErrorPage bool
+	OutputFilterErrorPage     bool
 	OutputFilterPageType      goflags.StringSlice
 	FilterOutDuplicates       bool
 	OutputFilterContentLength string
@@ -287,6 +287,9 @@ type Options struct {
 	RateLimitMinute           int
 	Probe                     bool
 	Resume                    bool
+	RetryRounds               int
+	RetryDelay                int
+	RetryTimeout              int
 	resumeCfg                 *ResumeCfg
 	Exclude                   goflags.StringSlice
 	HostMaxErrors             int
@@ -578,6 +581,9 @@ func ParseOptions() *Options {
 		flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"),
 		flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to save in bytes"),
 		flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to read in bytes"),
+		flagSet.IntVar(&options.RetryRounds, "retry-rounds", 0, "number of retry rounds for HTTP 429 responses (Too Many Requests)"),
+		flagSet.IntVar(&options.RetryDelay, "retry-delay", 500, "fallback delay in ms when Retry-After header is absent (HTTP 429)"),
+		flagSet.IntVar(&options.RetryTimeout, "retry-timeout", 30, "max total time in seconds for retry rounds (HTTP 429)"),
 	)
 
 	flagSet.CreateGroup("cloud", "Cloud",
@@ -840,6 +846,15 @@ func (options *Options) ValidateOptions() error {
 		options.Threads = defaultThreads
 	}
 
+	if options.RetryRounds > 0 {
+		if options.RetryDelay <= 0 {
+			return errors.New(fmt.Sprintf("invalid retry-delay: must be >0 when retry-rounds=%d (got %d)", options.RetryRounds, options.RetryDelay))
+		}
+		if options.RetryTimeout <= 0 {
+			return errors.New(fmt.Sprintf("invalid retry-timeout: must be >0 when retry-rounds=%d (got %d)", options.RetryRounds, options.RetryTimeout))
+		}
+	}
+
 	return nil
 }