From a1022d4d3f98cde3e567269bc85c97249e6d8856 Mon Sep 17 00:00:00 2001 From: TBX3D <88289044+TBX3D@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:22:47 -0700 Subject: [PATCH 1/2] feat(js): expand secret rule bank with provider-prefixed and structural rules the 8-rule bank covered aws/github/slack/stripe/google/pem plus a generic keyword=value fallback, missing entire categories real recon tooling (trufflehog, gitleaks) treats as table stakes: git hosting PATs beyond github, ai provider keys, payment/messaging provider tokens, and anything requiring more than shape matching. adds ~19 near-zero-FP provider-prefixed rules (gitlab, npm, pypi, anthropic, openai, stripe restricted/webhook, square, sendgrid, mailgun, discord bot/webhook, slack webhook, new relic, cloudinary, github fine-grained pat), plus two rules that need real validation logic instead of just a regex shape: a jwt rule that decodes the header segment and confirms it's actually jwt json (not just three dotted base64 blobs), and a db connection-string rule that filters out placeholder passwords (password/changeme/example/...) so docs and .env.example files don't flood findings. --- internal/scan/js/secrets.go | 172 +++++++++++++++++++++++++++- internal/scan/js/secrets_test.go | 185 ++++++++++++++++++++++++++++++- 2 files changed, 355 insertions(+), 2 deletions(-) diff --git a/internal/scan/js/secrets.go b/internal/scan/js/secrets.go index 892b5fb2..3c408d55 100644 --- a/internal/scan/js/secrets.go +++ b/internal/scan/js/secrets.go @@ -13,6 +13,7 @@ package js import ( + "encoding/base64" "math" "regexp" "strings" @@ -39,11 +40,13 @@ const ( // secretRules is the credential regex bank. the matching group (or the whole // match when there's no group) is what gets reported; minEntropy gates the -// generic high-entropy rules so we don't flag every short literal. +// generic high-entropy rules so we don't flag every short literal. validate +// runs after the entropy gate for rules where shape alone isn't proof. var secretRules = []struct { name string re *regexp.Regexp minEntropy float64 + validate func(string) bool }{ { // aws access key ids are fixed-shape and unmistakable. @@ -88,6 +91,121 @@ var secretRules = []struct { re: regexp.MustCompile(`-{5}BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-{5}`), minEntropy: noEntropyGate, }, + { + // github fine-grained personal access tokens: github_pat_ then 82 chars. + name: "github fine-grained pat", + re: regexp.MustCompile(`\b(github_pat_[0-9A-Za-z_]{82})\b`), + minEntropy: noEntropyGate, + }, + { + // gitlab personal/project/group access tokens. + name: "gitlab access token", + re: regexp.MustCompile(`\b(glpat-[0-9A-Za-z_-]{20,64})\b`), + minEntropy: noEntropyGate, + }, + { + name: "npm access token", + re: regexp.MustCompile(`\b(npm_[0-9A-Za-z]{36})\b`), + minEntropy: noEntropyGate, + }, + { + // pypi tokens all share the pypi-AgEIcHlwaS5vcmc prefix, the base64 + // encoding of a fixed macaroon header, so it's effectively unforgeable. + name: "pypi api token", + re: regexp.MustCompile(`\b(pypi-AgEIcHlwaS5vcmc[0-9A-Za-z_-]{50,})\b`), + minEntropy: noEntropyGate, + }, + { + // anthropic api and admin keys. + name: "anthropic api key", + re: regexp.MustCompile(`\b(sk-ant-(?:api03|admin01)-[0-9A-Za-z_-]{80,120})\b`), + minEntropy: noEntropyGate, + }, + { + // legacy openai secret keys embed a fixed T3BlbkFJ marker (base64 for + // "OpenAI") between two random halves. + name: "openai api key", + re: regexp.MustCompile(`\b(sk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})\b`), + minEntropy: noEntropyGate, + }, + { + // current-generation project and service-account keys. + name: "openai project api key", + re: regexp.MustCompile(`\b(sk-(?:proj|svcacct)-[A-Za-z0-9_-]{20,})\b`), + minEntropy: noEntropyGate, + }, + { + name: "stripe restricted key", + re: regexp.MustCompile(`\b(rk_live_[0-9A-Za-z]{16,})\b`), + minEntropy: noEntropyGate, + }, + { + name: "stripe webhook secret", + re: regexp.MustCompile(`\b(whsec_[0-9A-Za-z]{32,})\b`), + minEntropy: noEntropyGate, + }, + { + // square access/oauth tokens. + name: "square access token", + re: regexp.MustCompile(`\b(sq0atp-[0-9A-Za-z_-]{22}|sq0csp-[0-9A-Za-z_-]{43})\b`), + minEntropy: noEntropyGate, + }, + { + name: "sendgrid api key", + re: regexp.MustCompile(`\b(SG\.[0-9A-Za-z_-]{22}\.[0-9A-Za-z_-]{43})\b`), + minEntropy: noEntropyGate, + }, + { + // mailgun api keys, key- then a 32-char hex blob. + name: "mailgun api key", + re: regexp.MustCompile(`\b(key-[0-9a-f]{32})\b`), + minEntropy: noEntropyGate, + }, + { + // discord bot tokens: base64 user id, a timestamp segment, then an HMAC. + name: "discord bot token", + re: regexp.MustCompile(`\b([MNOP][A-Za-z0-9_-]{23}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,38})\b`), + minEntropy: noEntropyGate, + }, + { + // discord incoming-webhook urls embed the secret in the path. + name: "discord webhook url", + re: regexp.MustCompile(`\b(discord(?:app)?\.com/api/webhooks/[0-9]{17,20}/[A-Za-z0-9_-]{60,68})`), + minEntropy: noEntropyGate, + }, + { + // slack incoming-webhook urls embed the secret in the path. + name: "slack webhook url", + re: regexp.MustCompile(`\b(hooks\.slack\.com/services/T[0-9A-Za-z]+/B[0-9A-Za-z]+/[0-9A-Za-z]{24})`), + minEntropy: noEntropyGate, + }, + { + name: "new relic license key", + re: regexp.MustCompile(`\b(NRAK-[A-Z0-9]{27})\b`), + minEntropy: noEntropyGate, + }, + { + // cloudinary connection urls carry the api key and secret in the userinfo. + name: "cloudinary url", + re: regexp.MustCompile(`\b(cloudinary://[0-9]{10,20}:[A-Za-z0-9_-]{20,}@[A-Za-z0-9_-]+)`), + minEntropy: noEntropyGate, + }, + { + // jwts have no fixed prefix; validate decodes the header to rule out + // arbitrary dotted base64url blobs. + name: "jwt", + re: regexp.MustCompile(`\b(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})\b`), + minEntropy: noEntropyGate, + validate: isStructuredJWT, + }, + { + // validate drops the countless doc/template examples that use a + // placeholder password. + name: "database connection string", + re: regexp.MustCompile(`\b((?:mongodb(?:\+srv)?|postgres(?:ql)?|mysql|redis|rediss|amqp|amqps)://[^:\s"'` + "`" + `/@]+:[^@\s"'` + "`" + `/]+@[^\s"'` + "`" + `]+)`), + minEntropy: noEntropyGate, + validate: hasRealConnStringPassword, + }, { // generic apikey/secret/token = "" assignments; the value is in // group 2 and only reported if it looks random (entropy gate). @@ -125,6 +243,11 @@ func ScanSecrets(content, srcURL string) []SecretMatch { continue } + // structural validation for rules whose shape alone isn't proof. + if rule.validate != nil && !rule.validate(value) { + continue + } + // dedupe per source so a key referenced twice is one finding. key := rule.name + "\x00" + value if _, ok := seen[key]; ok { @@ -148,6 +271,53 @@ func secretValue(groups []string) string { return strings.TrimSpace(groups[wholeMatchIndex]) } +// alg is mandatory per RFC 7519; requiring both fields keeps arbitrary +// dot-separated base64url blobs from being mistaken for a jwt. +const ( + jwtAlgField = `"alg"` + jwtTypField = `"typ"` +) + +// connStringPasswordRe pulls the userinfo password out of a scheme://user:pass@host +// connection string, for filtering placeholder credentials post-match. +var connStringPasswordRe = regexp.MustCompile(`://[^:@/\s]*:([^@/\s]+)@`) + +// placeholderPasswords are stand-ins that show up constantly in docs, sample +// configs and .env.example files; matching one means the string isn't a real +// leaked credential. +var placeholderPasswords = map[string]struct{}{ + "password": {}, "pass": {}, "passwd": {}, "xxxx": {}, "xxxxx": {}, + "changeme": {}, "yourpassword": {}, "example": {}, "test": {}, + "123456": {}, "secret": {}, "admin": {}, "root": {}, "pwd": {}, +} + +// isStructuredJWT confirms the header segment decodes to jwt-shaped json. +func isStructuredJWT(token string) bool { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return false + } + + header, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return false + } + + h := string(header) + return strings.Contains(h, jwtAlgField) && strings.Contains(h, jwtTypField) +} + +// hasRealConnStringPassword rejects known placeholder passwords. +func hasRealConnStringPassword(value string) bool { + m := connStringPasswordRe.FindStringSubmatch(value) + if len(m) < 2 { + return true + } + + _, placeholder := placeholderPasswords[strings.ToLower(m[1])] + return !placeholder +} + // shannonEntropy is the per-character shannon entropy (bits) of s, used to tell // random-looking secrets apart from plain words. empty input is zero entropy. func shannonEntropy(s string) float64 { diff --git a/internal/scan/js/secrets_test.go b/internal/scan/js/secrets_test.go index e4b7807b..44212bc4 100644 --- a/internal/scan/js/secrets_test.go +++ b/internal/scan/js/secrets_test.go @@ -14,6 +14,7 @@ package js import ( "fmt" + "strings" "testing" ) @@ -21,7 +22,7 @@ import ( // provider token literal in a committed file trips github push-protection (and // every other secret scanner) even though it's a test fixture. splitting it // keeps the literal out of source while ScanSecrets still sees the joined value. -const ( +var ( fakeAWSKey = "AKIA" + "IOSFODNN7EXAMPLE" fakeAWSSecret = "wJalrXUtnFEMI/K7MDENG/" + "bPxRfiCYEXAMPLEKEY" fakeGitHub = "ghp_" + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcd" @@ -30,6 +31,31 @@ const ( fakeGoogle = "AIza" + "SyA1B2C3D4E5F6G7H8I9J0K1L2M3N4O5P6Q" fakeGeneric = "x9Kq2Lm7Pz4Rt6Wv8Bn3Cd5Fg1Hj0As" fakePEM = "-----BEGIN RSA PRIVATE " + "KEY-----\nMIIEpAIB..." + + fakeGitHubFine = "github_pat_" + strings.Repeat("aB3dEfGh1j", 8) + "aB" + fakeGitLab = "glpat-" + "aB3dEfGh1jKlMn0pQrSt" + fakeNpm = "npm_" + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcd" + fakePyPI = "pypi-AgEIcHlwaS5vcmc" + strings.Repeat("1jKlMn0p", 7) + fakeAnthropic = "sk-ant-api03-" + strings.Repeat("aB3dEfGh1j", 9) + fakeOpenAILeg = "sk-" + "aB3dEfGh1jKlMn0pQrSt" + "T3BlbkFJ" + "uVwXyZ012345abcdefgh" + fakeOpenAIProj = "sk-proj-" + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcd" + fakeStripeRk = "rk_live_" + "4eC39HqLyjWDarjtT1zdp7dc" + fakeStripeWH = "whsec_" + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcd" + fakeSquare = "sq0atp-" + "aB3dEfGh1jKlMn0pQrSt-U" + fakeSendGrid = "SG." + "aB3dEfGh1jKlMn0pQrSt-U" + "." + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcdefghijk" + fakeMailgun = "key-" + "0123456789abcdef0123456789abcdef" + fakeDiscordBot = "M" + "TIzNDU2Nzg5MDEyMzQ1Njc4" + "." + "GaBcDe" + "." + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcdef" + fakeDiscordHook = "discord.com/api/webhooks/" + "123456789012345678" + "/" + strings.Repeat("aB3dEfGh1j", 6) + "abcdefgh" + fakeSlackHook = "hooks.slack.com/services/T" + "0123ABCD" + "/B" + "0123ABCD" + "/" + "aB3dEfGh1jKlMn0pQrStUvWx" + fakeNewRelic = "NRAK-" + "AB3DEFGH1JKLMN0PQRSTUVWXYZZ" + fakeCloudinary = "cloudinary://" + "123456789012345" + ":" + "aB3dEfGh1jKlMn0pQrStUvWxYz" + "@my-cloud" + fakeMongoURI = "mongodb+srv://" + "dbadmin" + ":" + "tR7q!zK2vLp9xC" + "@cluster0.example.mongodb.net/prod" + fakeMongoPlace = "mongodb://" + "user" + ":" + "password" + "@localhost:27017/app" + + // a real jwt (rfc 7519 example header/payload), signature is dummy. + fakeJWT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" + + ".eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIn0" + + ".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" ) func TestScanSecrets(t *testing.T) { @@ -79,6 +105,111 @@ func TestScanSecrets(t *testing.T) { content: fmt.Sprintf(`aws_secret_access_key=%q`, fakeAWSSecret), wantRule: "aws secret access key", }, + { + name: "github fine-grained pat", + content: fmt.Sprintf(`token: %q`, fakeGitHubFine), + wantRule: "github fine-grained pat", + }, + { + name: "gitlab access token", + content: fmt.Sprintf(`GITLAB_TOKEN=%q`, fakeGitLab), + wantRule: "gitlab access token", + }, + { + name: "npm access token", + content: fmt.Sprintf(`_authToken=%q`, fakeNpm), + wantRule: "npm access token", + }, + { + name: "pypi api token", + content: fmt.Sprintf(`password = %q`, fakePyPI), + wantRule: "pypi api token", + }, + { + name: "anthropic api key", + content: fmt.Sprintf(`ANTHROPIC_API_KEY=%q`, fakeAnthropic), + wantRule: "anthropic api key", + }, + { + name: "openai legacy api key", + content: fmt.Sprintf(`OPENAI_API_KEY=%q`, fakeOpenAILeg), + wantRule: "openai api key", + }, + { + name: "openai project api key", + content: fmt.Sprintf(`OPENAI_API_KEY=%q`, fakeOpenAIProj), + wantRule: "openai project api key", + }, + { + name: "stripe restricted key", + content: fmt.Sprintf(`var rk = %q;`, fakeStripeRk), + wantRule: "stripe restricted key", + }, + { + name: "stripe webhook secret", + content: fmt.Sprintf(`endpointSecret = %q`, fakeStripeWH), + wantRule: "stripe webhook secret", + }, + { + name: "square access token", + content: fmt.Sprintf(`squareToken = %q`, fakeSquare), + wantRule: "square access token", + }, + { + name: "sendgrid api key", + content: fmt.Sprintf(`SENDGRID_API_KEY=%q`, fakeSendGrid), + wantRule: "sendgrid api key", + }, + { + name: "mailgun api key", + content: fmt.Sprintf(`MAILGUN_KEY=%q`, fakeMailgun), + wantRule: "mailgun api key", + }, + { + name: "discord bot token", + content: fmt.Sprintf(`client.login(%q)`, fakeDiscordBot), + wantRule: "discord bot token", + }, + { + name: "discord webhook url", + content: fmt.Sprintf(`fetch("https://%s")`, fakeDiscordHook), + wantRule: "discord webhook url", + }, + { + name: "slack webhook url", + content: fmt.Sprintf(`fetch("https://%s")`, fakeSlackHook), + wantRule: "slack webhook url", + }, + { + name: "new relic license key", + content: fmt.Sprintf(`NEW_RELIC_LICENSE_KEY=%q`, fakeNewRelic), + wantRule: "new relic license key", + }, + { + name: "cloudinary url", + content: fmt.Sprintf(`CLOUDINARY_URL=%q`, fakeCloudinary), + wantRule: "cloudinary url", + }, + { + name: "jwt with valid header", + content: fmt.Sprintf(`const token = %q;`, fakeJWT), + wantRule: "jwt", + }, + { + name: "three dotted base64 blobs without a jwt header are not flagged as a jwt", + content: `const chunkHash = "eyJmb28iOiJiYXIiLCJraWQiOjEyM30.notactuallyjsonatall.dGhpcyBpcyBub3QgYSBqd3Q";`, + wantNone: true, + }, + { + name: "mongodb uri with real credentials", + content: fmt.Sprintf(`const uri = %q;`, fakeMongoURI), + wantRule: "database connection string", + }, + { + name: "mongodb uri with placeholder password not flagged", + content: fmt.Sprintf(`// example: %s`, fakeMongoPlace), + wantNone: true, + }, { // low-entropy assignment is a placeholder, not a real secret. name: "low entropy generic assignment not flagged", @@ -125,6 +256,58 @@ func TestScanSecrets(t *testing.T) { } } +// TestScanSecretsCoverageDelta proves every newly added rule fires against +// its own fixture in a single bundled scan. +func TestScanSecretsCoverageDelta(t *testing.T) { + bundle := strings.Join([]string{ + fmt.Sprintf(`token: %q`, fakeGitHubFine), + fmt.Sprintf(`GITLAB_TOKEN=%q`, fakeGitLab), + fmt.Sprintf(`_authToken=%q`, fakeNpm), + fmt.Sprintf(`password = %q`, fakePyPI), + fmt.Sprintf(`ANTHROPIC_API_KEY=%q`, fakeAnthropic), + fmt.Sprintf(`OPENAI_API_KEY=%q`, fakeOpenAILeg), + fmt.Sprintf(`OPENAI_API_KEY=%q`, fakeOpenAIProj), + fmt.Sprintf(`var rk = %q;`, fakeStripeRk), + fmt.Sprintf(`endpointSecret = %q`, fakeStripeWH), + fmt.Sprintf(`squareToken = %q`, fakeSquare), + fmt.Sprintf(`SENDGRID_API_KEY=%q`, fakeSendGrid), + fmt.Sprintf(`MAILGUN_KEY=%q`, fakeMailgun), + fmt.Sprintf(`client.login(%q)`, fakeDiscordBot), + fmt.Sprintf(`fetch("https://%s")`, fakeDiscordHook), + fmt.Sprintf(`fetch("https://%s")`, fakeSlackHook), + fmt.Sprintf(`NEW_RELIC_LICENSE_KEY=%q`, fakeNewRelic), + fmt.Sprintf(`CLOUDINARY_URL=%q`, fakeCloudinary), + fmt.Sprintf(`const token = %q;`, fakeJWT), + fmt.Sprintf(`const uri = %q;`, fakeMongoURI), + }, "\n") + + wantRules := []string{ + "github fine-grained pat", "gitlab access token", "npm access token", + "pypi api token", "anthropic api key", "openai api key", + "openai project api key", "stripe restricted key", "stripe webhook secret", + "square access token", "sendgrid api key", "mailgun api key", + "discord bot token", "discord webhook url", "slack webhook url", + "new relic license key", "cloudinary url", "jwt", "database connection string", + } + + // some fixtures also trip the generic keyword=value rule, so assert + // presence per rule rather than an exact count. + got := ScanSecrets(bundle, "https://example.com/app.js") + if len(got) < len(wantRules) { + t.Fatalf("expected at least %d matches (one per new rule), got %d: %+v", len(wantRules), len(got), got) + } + + foundRules := make(map[string]bool, len(got)) + for _, m := range got { + foundRules[m.Rule] = true + } + for _, want := range wantRules { + if !foundRules[want] { + t.Errorf("rule %q did not fire against its own fixture", want) + } + } +} + func TestScanSecretsDedupesWithinSource(t *testing.T) { // the same key referenced twice in one file is one finding. content := fmt.Sprintf(`a = %q; b = %q;`, fakeAWSKey, fakeAWSKey) From 9aa7489357ef3b2be8668125fa85b69a6e15630c Mon Sep 17 00:00:00 2001 From: TBX3D <88289044+TBX3D@users.noreply.github.com> Date: Thu, 9 Jul 2026 18:49:07 -0700 Subject: [PATCH 2/2] fix(js): drop thin gitlab and square token rule comments --- internal/scan/js/secrets.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/scan/js/secrets.go b/internal/scan/js/secrets.go index 3c408d55..a92e5f67 100644 --- a/internal/scan/js/secrets.go +++ b/internal/scan/js/secrets.go @@ -98,7 +98,6 @@ var secretRules = []struct { minEntropy: noEntropyGate, }, { - // gitlab personal/project/group access tokens. name: "gitlab access token", re: regexp.MustCompile(`\b(glpat-[0-9A-Za-z_-]{20,64})\b`), minEntropy: noEntropyGate, @@ -145,7 +144,6 @@ var secretRules = []struct { minEntropy: noEntropyGate, }, { - // square access/oauth tokens. name: "square access token", re: regexp.MustCompile(`\b(sq0atp-[0-9A-Za-z_-]{22}|sq0csp-[0-9A-Za-z_-]{43})\b`), minEntropy: noEntropyGate,