diff --git a/README.md b/README.md index 5dddd737..d70d1047 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ CONFIGURATIONS: -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443 -ztls use ztls library with autofallback to standard one for tls13 -no-decode avoid decoding body - -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization + -tlsi, -tls-impersonate string enable experimental client hello (ja3) tls impersonation (random, chrome, or ja3 full string) -no-stdin Disable Stdin processing -hae, -http-api-endpoint string experimental http api endpoint -sf, -secret-file string path to secret file for authentication diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index 34a53b4c..0cdf3268 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -19,5 +19,5 @@ scanme.sh {{binary}} -silent -ztls scanme.sh {{binary}} -silent -jarm https://scanme.sh?a=1*1 {{binary}} -silent https://scanme.sh:443 {{binary}} -asn -scanme.sh {{binary}} -silent -tls-impersonate +scanme.sh {{binary}} -silent -tls-impersonate random example.com {{binary}} -silent -bp -strip \ No newline at end of file diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 82198887..c4050da3 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -16,6 +16,7 @@ import ( "github.com/microcosm-cc/bluemonday" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fastdialer/fastdialer/ja3" "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/networkpolicy" @@ -139,12 +140,7 @@ func New(options *Options) (*HTTPX, error) { } transport := &http.Transport{ DialContext: httpx.Dialer.Dial, - DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - if options.TlsImpersonate { - return httpx.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil) - } - return httpx.Dialer.DialTLS(ctx, network, addr) - }, + DialTLSContext: httpx.buildTLSDialer(options), MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, @@ -216,6 +212,36 @@ func New(options *Options) (*HTTPX, error) { return httpx, nil } +func (h *HTTPX) buildTLSDialer(options *Options) func(ctx context.Context, network, addr string) (net.Conn, error) { + if options.TlsImpersonate == "" { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return h.Dialer.DialTLS(ctx, network, addr) + } + } + + tlsCfg := &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10} + + strategy, identity := resolveImpersonateStrategy(options.TlsImpersonate) + + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return h.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsCfg, strategy, identity) + } +} + +func resolveImpersonateStrategy(value string) (impersonate.Strategy, *impersonate.Identity) { + switch strings.ToLower(value) { + case "", "chrome": + return impersonate.Chrome, nil + default: + spec, err := ja3.ParseWithJa3(value) + if err != nil { + return impersonate.Chrome, nil + } + identity := impersonate.Identity(*spec) + return impersonate.Custom, &identity + } +} + // Do http request func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Response, error) { timeStart := time.Now() diff --git a/common/httpx/option.go b/common/httpx/option.go index fb108729..bd47f167 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -58,7 +58,7 @@ type Options struct { Resolvers []string customCookies []*http.Cookie SniName string - TlsImpersonate bool + TlsImpersonate string NetworkPolicy *networkpolicy.NetworkPolicy CDNCheckClient *cdncheck.Client Protocol Proto diff --git a/common/httpx/tls_impersonate_test.go b/common/httpx/tls_impersonate_test.go new file mode 100644 index 00000000..773f7bef --- /dev/null +++ b/common/httpx/tls_impersonate_test.go @@ -0,0 +1,371 @@ +package httpx + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "sync" + "testing" + "time" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" + "github.com/stretchr/testify/require" +) + +// capturedHello holds the ClientHello details captured by the test TLS server. +type capturedHello struct { + CipherSuites []uint16 + SupportedCurves []tls.CurveID + ServerName string + SupportedProtos []string +} + +// startTLSServer creates a local TLS server that captures ClientHello info +// from each incoming connection. It returns the listener address and a function +// to retrieve the most recently captured hello. +func startTLSServer(t *testing.T) (string, func() *capturedHello) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + } + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + + cert := tls.Certificate{ + Certificate: [][]byte{certDER}, + PrivateKey: key, + } + + var mu sync.Mutex + var latest *capturedHello + + tlsCfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + GetConfigForClient: func(info *tls.ClientHelloInfo) (*tls.Config, error) { + hello := &capturedHello{ + CipherSuites: info.CipherSuites, + SupportedCurves: info.SupportedCurves, + ServerName: info.ServerName, + SupportedProtos: info.SupportedProtos, + } + mu.Lock() + latest = hello + mu.Unlock() + return nil, nil + }, + } + + ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsCfg) + require.NoError(t, err) + t.Cleanup(func() { + _ = ln.Close() + }) + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer func() { + _ = conn.Close() + }() + buf := make([]byte, 1) + _, _ = conn.Read(buf) + }() + } + }() + + getHello := func() *capturedHello { + mu.Lock() + defer mu.Unlock() + return latest + } + + return ln.Addr().String(), getHello +} + +// --- Unit tests for resolveImpersonateStrategy --- + +func TestResolveImpersonateStrategy(t *testing.T) { + t.Run("empty defaults to chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("chrome") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("chrome case insensitive", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("CHROME") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("valid ja3 string", func(t *testing.T) { + ja3str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + strategy, identity := resolveImpersonateStrategy(ja3str) + require.Equal(t, impersonate.Custom, strategy) + require.NotNil(t, identity) + }) + + t.Run("invalid ja3 falls back to chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("not-a-ja3-string") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) + + t.Run("partial ja3 falls back to chrome", func(t *testing.T) { + strategy, identity := resolveImpersonateStrategy("771,4865") + require.Equal(t, impersonate.Chrome, strategy) + require.Nil(t, identity) + }) +} + +// Integration tests with local TLS server + +func TestTLSImpersonate_DefaultGoTLS(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.None, nil, + ) + require.NoError(t, err) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + require.NotEmpty(t, hello.CipherSuites, "default Go TLS should have cipher suites") +} + +func TestTLSImpersonate_Chrome(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.Chrome, nil, + ) + require.NoError(t, err) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + require.NotEmpty(t, hello.CipherSuites) + hasGrease := false + for _, cs := range hello.CipherSuites { + if cs&0x0f0f == 0x0a0a { + hasGrease = true + break + } + } + require.True(t, hasGrease, "Chrome impersonation should include GREASE cipher suite values") +} + +func TestTLSImpersonate_CustomJA3(t *testing.T) { + addr, getHello := startTLSServer(t) + + ja3Str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + strategy, identity := resolveImpersonateStrategy(ja3Str) + require.Equal(t, impersonate.Custom, strategy) + require.NotNil(t, identity) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + strategy, identity, + ) + require.NoError(t, err) + require.NotNil(t, conn) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + + require.Equal(t, []uint16{49195, 49196}, hello.CipherSuites, + "custom JA3 should contain exactly the specified cipher suites") + + expectedCurves := []tls.CurveID{23, 24} + require.Equal(t, expectedCurves, hello.SupportedCurves, + "custom JA3 should contain exactly the specified curves") +} + +func TestTLSImpersonate_ChromeDiffersFromDefault(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.None, nil, + ) + require.NoError(t, err) + _ = conn.Close() + defaultHello := getHello() + + conn, err = fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.Chrome, nil, + ) + require.NoError(t, err) + _ = conn.Close() + chromeHello := getHello() + + require.NotNil(t, defaultHello) + require.NotNil(t, chromeHello) + + require.NotEqual(t, defaultHello.CipherSuites, chromeHello.CipherSuites, + "Chrome impersonation should produce different cipher suites than default Go TLS") +} + +func TestTLSImpersonate_CustomJA3DiffersFromDefault(t *testing.T) { + addr, getHello := startTLSServer(t) + + opts := fastdialer.DefaultOptions + opts.EnableFallback = false + fd, err := fastdialer.NewDialer(opts) + require.NoError(t, err) + defer fd.Close() + + // Default (no impersonation) + conn, err := fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + impersonate.None, nil, + ) + require.NoError(t, err) + _ = conn.Close() + defaultHello := getHello() + + ja3Str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + strategy, identity := resolveImpersonateStrategy(ja3Str) + require.Equal(t, impersonate.Custom, strategy) + + conn, err = fd.DialTLSWithConfigImpersonate( + context.Background(), "tcp", addr, + &tls.Config{InsecureSkipVerify: true}, + strategy, identity, + ) + require.NoError(t, err) + _ = conn.Close() + customHello := getHello() + + require.NotNil(t, defaultHello) + require.NotNil(t, customHello) + + require.NotEqual(t, defaultHello.CipherSuites, customHello.CipherSuites, + "custom JA3 should produce different cipher suites than default Go TLS") +} + +func TestTLSImpersonate_EndToEnd_HTTPX(t *testing.T) { + addr, getHello := startTLSServer(t) + + tests := []struct { + name string + strategy string + wantErr bool + }{ + {"disabled", "", false}, + {"chrome", "chrome", false}, + {"ja3", "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options := DefaultOptions + options.TlsImpersonate = tt.strategy + + ht, err := New(&options) + require.NoError(t, err) + + dialer := ht.buildTLSDialer(&options) + conn, err := dialer(context.Background(), "tcp", addr) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.NotNil(t, conn) + _ = conn.Close() + + if tt.strategy != "" { + hello := getHello() + require.NotNil(t, hello) + require.NotEmpty(t, hello.CipherSuites) + } + }) + } +} + +func TestTLSImpersonate_EndToEnd_JA3(t *testing.T) { + addr, getHello := startTLSServer(t) + + ja3Str := "771,49195-49196,0-23-65281-10-11-35-16-5-13-18,23-24,0" + options := DefaultOptions + options.TlsImpersonate = ja3Str + + ht, err := New(&options) + require.NoError(t, err) + + dialer := ht.buildTLSDialer(&options) + conn, err := dialer(context.Background(), "tcp", addr) + require.NoError(t, err) + require.NotNil(t, conn) + _ = conn.Close() + + hello := getHello() + require.NotNil(t, hello) + + require.Equal(t, []uint16{49195, 49196}, hello.CipherSuites, + "JA3 end-to-end cipher suites should match exactly") + require.Equal(t, []tls.CurveID{23, 24}, hello.SupportedCurves, + "JA3 end-to-end curves should match exactly") +} diff --git a/go.mod b/go.mod index 529c81c1..3894c697 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/projectdiscovery/cdncheck v1.2.28 github.com/projectdiscovery/clistats v0.1.1 github.com/projectdiscovery/dsl v0.8.14 - github.com/projectdiscovery/fastdialer v0.5.5 + github.com/projectdiscovery/fastdialer v0.5.6-0.20260322114839-243754103eca github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.74 diff --git a/go.sum b/go.sum index 22e85fe1..c4574382 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.8.14 h1:g9szcXk2RRdVf2rsHEzbTXOPxiny3haKonSncU6pg2w= github.com/projectdiscovery/dsl v0.8.14/go.mod h1:LYImt/EiBzqTWG1RswT3Yl0DZbfjUP93Nvq2Z/G7dcE= -github.com/projectdiscovery/fastdialer v0.5.5 h1:KXmGuR1Op37umSvx4B0vxVSuC2a2DqD1oMqZ5l2bLEU= -github.com/projectdiscovery/fastdialer v0.5.5/go.mod h1:QxvCe02Jii+j8vA3hWYkymgZIY8cqMgs2s3Jbz6mvbs= +github.com/projectdiscovery/fastdialer v0.5.6-0.20260322114839-243754103eca h1:g7uHD+yWd6owMn5GFnCLuQN+f1P11kSK508OyKIlIyI= +github.com/projectdiscovery/fastdialer v0.5.6-0.20260322114839-243754103eca/go.mod h1:QxvCe02Jii+j8vA3hWYkymgZIY8cqMgs2s3Jbz6mvbs= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= diff --git a/runner/options.go b/runner/options.go index d12a7dad..385e0de7 100644 --- a/runner/options.go +++ b/runner/options.go @@ -331,7 +331,7 @@ type Options struct { NoDecode bool Screenshot bool UseInstalledChrome bool - TlsImpersonate bool + TlsImpersonate string DisableStdin bool HttpApiEndpoint string NoScreenshotBytes bool @@ -547,7 +547,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https://host:443"), flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"), flagSet.BoolVar(&options.NoDecode, "no-decode", false, "avoid decoding body"), - flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), + flagSet.StringVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", "", "enable experimental client hello (ja3) tls impersonation (random, chrome, or ja3 full string)"), flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"), flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), flagSet.StringVarP(&options.SecretFile, "secret-file", "sf", "", "path to the secret file for authentication"),