secretscan.py

#!/usr/bin/env python3
"""
secretscan -- Find leaked secrets in your codebase. Zero deps.

Scans source files for API keys, tokens, passwords, private keys, and
high-entropy strings. CI/CD friendly (exit 1 if secrets found).

Usage:
    py secretscan.py .                              # Scan current directory
    py secretscan.py src/ tests/                    # Scan specific dirs
    py secretscan.py . --format json                # JSON output
    py secretscan.py . -v                           # Show matched values
    py secretscan.py . --allow .secretscan-allow    # Allowlist file
    py secretscan.py . --entropy 4.5                # Adjust entropy threshold
    py secretscan.py . -g "*.py" "*.js"             # Only scan specific files
"""

import argparse
import math
import os
import re
import sys
from collections import Counter
from pathlib import Path

# Colors
RESET = "\033[0m"
BOLD = "\033[1m"
DIM = "\033[2m"
RED = "\033[31m"
YELLOW = "\033[33m"
GREEN = "\033[32m"
CYAN = "\033[36m"

# Directories to always skip
SKIP_DIRS = {
    ".git", "__pycache__", "node_modules", ".venv", "venv", "env",
    ".tox", ".eggs", "dist", "build", ".next", ".nuxt", "vendor",
    ".idea", ".vscode", ".vs", "coverage", ".cache", ".terraform",
    "target", "bin", "obj",
}

# File extensions to skip (binary/media)
SKIP_EXTS = {
    ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".svg", ".webp",
    ".mp3", ".mp4", ".avi", ".mov", ".mkv", ".wav", ".flac",
    ".zip", ".tar", ".gz", ".bz2", ".xz", ".rar", ".7z",
    ".exe", ".dll", ".so", ".dylib", ".bin", ".dat",
    ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt",
    ".woff", ".woff2", ".ttf", ".eot", ".otf",
    ".pyc", ".pyo", ".class", ".o", ".a",
    ".lock", ".sum",
}

# Secret patterns: (name, regex, severity)
SECRET_PATTERNS = [
    # AWS
    ("AWS Access Key", r"(?:^|[^A-Za-z0-9])(?:AKIA[0-9A-Z]{16})(?:[^A-Za-z0-9]|$)", "high"),
    ("AWS Secret Key", r"(?:aws_secret_access_key|aws_secret)\s*[=:]\s*['\"]?([A-Za-z0-9/+=]{40})['\"]?", "high"),
    # Generic API keys
    ("API Key Assignment", r"(?:api[_-]?key|apikey|api[_-]?secret)\s*[=:]\s*['\"]([a-zA-Z0-9_\-]{20,})['\"]", "high"),
    ("Bearer Token", r"(?:bearer|token|auth)\s*[=:]\s*['\"]([a-zA-Z0-9_\-.]{20,})['\"]", "medium"),
    # GitHub
    ("GitHub Token", r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36,}", "high"),
    ("GitHub Fine-grained PAT", r"github_pat_[A-Za-z0-9_]{82,}", "high"),
    # OpenAI / Anthropic
    ("OpenAI API Key", r"sk-[a-zA-Z0-9]{20,}", "high"),
    ("Anthropic API Key", r"sk-ant-[a-zA-Z0-9_\-]{20,}", "high"),
    # Stripe
    ("Stripe Secret Key", r"sk_live_[a-zA-Z0-9]{20,}", "high"),
    ("Stripe Publishable Key", r"pk_live_[a-zA-Z0-9]{20,}", "medium"),
    # Database
    ("Database URL", r"(?:postgres|mysql|mongodb|redis|amqp)://[^\s'\"]{10,}", "high"),
    ("Connection String", r"(?:Server|Data Source)\s*=\s*[^;]+;.*(?:Password|Pwd)\s*=\s*[^;]+", "high"),
    # Private Keys
    ("Private Key", r"-----BEGIN (?:RSA |DSA |EC |OPENSSH )?PRIVATE KEY-----", "critical"),
    ("PGP Private Key", r"-----BEGIN PGP PRIVATE KEY BLOCK-----", "critical"),
    # Generic passwords
    ("Password Assignment", r"(?:password|passwd|pwd|secret)\s*[=:]\s*['\"]([^'\"]{8,})['\"]", "high"),
    ("Hardcoded Password", r"(?:password|passwd|pwd)\s*=\s*['\"](?![\s{<$])[^'\"]{8,}['\"]", "high"),
    # JWT
    ("JWT Token", r"eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}", "medium"),
    # Slack
    ("Slack Token", r"xox[bpors]-[0-9a-zA-Z]{10,}", "high"),
    ("Slack Webhook", r"https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[a-zA-Z0-9]+", "high"),
    # Google
    ("Google API Key", r"AIza[0-9A-Za-z_\-]{35}", "high"),
    ("Google OAuth Secret", r"(?:client_secret)\s*[=:]\s*['\"]([a-zA-Z0-9_\-]{24,})['\"]", "medium"),
    # Twilio
    ("Twilio API Key", r"SK[a-f0-9]{32}", "medium"),
    # SendGrid
    ("SendGrid API Key", r"SG\.[a-zA-Z0-9_\-]{22,}\.[a-zA-Z0-9_\-]{43,}", "high"),
    # Heroku
    ("Heroku API Key", r"(?:heroku_api_key|HEROKU_API_KEY)\s*[=:]\s*['\"]?([a-f0-9-]{36})['\"]?", "high"),
    # NPM
    ("NPM Token", r"//registry\.npmjs\.org/:_authToken=([a-f0-9-]{36})", "high"),
    # Generic high-entropy hex
    ("Hex Secret (40 char)", r"(?:secret|token|key|pass)\s*[=:]\s*['\"]([a-f0-9]{40})['\"]", "medium"),
]


def color_supported() -> bool:
    if os.environ.get("NO_COLOR"):
        return False
    if sys.platform == "win32":
        return bool(os.environ.get("TERM") or os.environ.get("WT_SESSION"))
    return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()


USE_COLOR = color_supported()


def c(code: str, text: str) -> str:
    return f"{code}{text}{RESET}" if USE_COLOR else text


def severity_color(sev: str) -> str:
    return {"critical": RED + BOLD, "high": RED, "medium": YELLOW}.get(sev, DIM)


def shannon_entropy(s: str) -> float:
    """Calculate Shannon entropy of a string."""
    if not s:
        return 0.0
    freq = Counter(s)
    length = len(s)
    return -sum((count / length) * math.log2(count / length) for count in freq.values())


def find_high_entropy(line: str, threshold: float = 4.5) -> list[tuple[str, str, float]]:
    """Find high-entropy strings in a line (potential secrets)."""
    findings = []
    # Look for quoted strings or assignments with long values
    for match in re.finditer(r"""[=:]\s*['"]([a-zA-Z0-9+/=_\-]{20,})['"]""", line):
        candidate = match.group(1)
        ent = shannon_entropy(candidate)
        if ent >= threshold and len(candidate) >= 20:
            findings.append(("High Entropy String", candidate, ent))
    return findings


def should_scan(filepath: Path, glob_patterns: list[str] | None) -> bool:
    """Check if a file should be scanned."""
    if filepath.suffix.lower() in SKIP_EXTS:
        return False
    if filepath.name.startswith("."):
        return True  # scan dotfiles like .env
    if glob_patterns:
        import fnmatch
        return any(fnmatch.fnmatch(filepath.name, p) for p in glob_patterns)
    return True


def scan_file(filepath: Path, compiled_patterns: list, entropy_threshold: float,
              allowlist: set) -> list[dict]:
    """Scan a single file for secrets."""
    findings = []
    try:
        content = filepath.read_text(encoding="utf-8", errors="replace")
    except Exception:
        return []

    for lineno, line in enumerate(content.splitlines(), 1):
        stripped = line.strip()
        if not stripped or stripped.startswith("#") and len(stripped) < 200:
            continue  # Skip pure comment short lines (but scan long ones)

        # Check against patterns
        for name, pattern, severity in compiled_patterns:
            for match in pattern.finditer(line):
                matched_text = match.group(0)[:80]
                # Check allowlist
                if any(a in line for a in allowlist):
                    continue
                findings.append({
                    "file": str(filepath),
                    "line": lineno,
                    "rule": name,
                    "severity": severity,
                    "match": matched_text,
                    "context": stripped[:120],
                })

        # Entropy check
        if entropy_threshold > 0:
            for name, candidate, ent in find_high_entropy(line, entropy_threshold):
                if any(a in line for a in allowlist):
                    continue
                findings.append({
                    "file": str(filepath),
                    "line": lineno,
                    "rule": f"{name} (entropy: {ent:.1f})",
                    "severity": "medium",
                    "match": candidate[:60],
                    "context": stripped[:120],
                })

    return findings


def collect_files(paths: list[str], recursive: bool = True,
                  glob_patterns: list[str] | None = None) -> list[Path]:
    """Collect files to scan."""
    files = []
    for path_str in paths:
        p = Path(path_str)
        if p.is_file():
            if should_scan(p, glob_patterns):
                files.append(p)
        elif p.is_dir():
            for root, dirs, filenames in os.walk(p):
                # Prune skipped directories
                dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
                for fname in filenames:
                    fp = Path(root) / fname
                    if should_scan(fp, glob_patterns):
                        files.append(fp)
                if not recursive:
                    break
    return files


def load_allowlist(path: str | None) -> set:
    """Load allowlist patterns (one per line)."""
    if not path:
        return set()
    p = Path(path)
    if not p.exists():
        return set()
    return {line.strip() for line in p.read_text(encoding="utf-8").splitlines()
            if line.strip() and not line.startswith("#")}


def print_findings(findings: list[dict], show_values: bool = False):
    """Print findings in text format."""
    if not findings:
        print(f"\n  {c(GREEN, 'No secrets found.')}\n")
        return

    # Group by file
    by_file: dict[str, list] = {}
    for f in findings:
        by_file.setdefault(f["file"], []).append(f)

    print()
    for filepath, file_findings in sorted(by_file.items()):
        print(c(CYAN, f"  {filepath}"))
        for f in file_findings:
            sev = f["severity"]
            sc = c(severity_color(sev), sev.upper())
            print(f"    Line {f['line']:>4}  {sc:<22}  {f['rule']}")
            if show_values:
                print(c(DIM, f"             {f['match']}"))
        print()

    # Summary
    by_severity = Counter(f["severity"] for f in findings)
    parts = []
    for sev in ["critical", "high", "medium"]:
        count = by_severity.get(sev, 0)
        if count:
            parts.append(c(severity_color(sev), f"{count} {sev}"))
    print(f"  {c(BOLD, 'Found:')} {', '.join(parts)} ({len(findings)} total in {len(by_file)} file(s))")
    print()


def main():
    parser = argparse.ArgumentParser(
        description="secretscan -- find leaked secrets in your codebase",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument("paths", nargs="+", help="File(s) or directory(ies) to scan")
    parser.add_argument("--format", choices=["text", "json"], default="text", help="Output format")
    parser.add_argument("-v", "--verbose", action="store_true", help="Show matched secret values")
    parser.add_argument("--entropy", type=float, default=4.5,
                        help="Shannon entropy threshold for high-entropy detection (0 to disable, default: 4.5)")
    parser.add_argument("--allow", help="Path to allowlist file (patterns to ignore, one per line)")
    parser.add_argument("-g", "--glob", nargs="+", help="Only scan files matching glob pattern(s)")
    parser.add_argument("--no-entropy", action="store_true", help="Disable entropy-based detection")

    args = parser.parse_args()

    entropy_threshold = 0 if args.no_entropy else args.entropy
    allowlist = load_allowlist(args.allow)

    # Compile patterns
    compiled = []
    for name, pattern, severity in SECRET_PATTERNS:
        try:
            compiled.append((name, re.compile(pattern, re.IGNORECASE), severity))
        except re.error:
            pass

    # Collect files
    files = collect_files(args.paths, glob_patterns=args.glob)

    if args.format == "text":
        print(f"\n  {c(BOLD, 'secretscan')}")
        print(c(DIM, f"  Scanning {len(files)} file(s)..."))

    # Scan
    all_findings = []
    for filepath in files:
        findings = scan_file(filepath, compiled, entropy_threshold, allowlist)
        all_findings.extend(findings)

    # Output
    if args.format == "json":
        import json
        print(json.dumps(all_findings, indent=2))
    else:
        print_findings(all_findings, show_values=args.verbose)

    # Exit code
    if all_findings:
        sys.exit(1)


if __name__ == "__main__":
    main()