diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 7894e8f..0000000 --- a/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -.git -.github -docs -test -scripts/sync.sh -**/__pycache__ -**/*.pyc -.venv -venv diff --git a/.github/cliff.toml b/.github/cliff.toml deleted file mode 100644 index edbe144..0000000 --- a/.github/cliff.toml +++ /dev/null @@ -1,46 +0,0 @@ -# git-cliff config - ported from pond's release-plz [changelog] so releases -# read the same: emoji-grouped sections, scope-bolded entries, PR links, -# breaking markers, and a Full Changelog compare footer. The `` -# prefixes order the sections (group_by sorts alphabetically; GitHub renders -# the HTML comments invisibly). - -[changelog] -# Single trailing newline: git-cliff emits one more before the first release body. -header = """# Changelog -""" -# trim=false with exact template whitespace: unlike release-plz (which splices -# sections into the changelog), git-cliff concatenates rendered bodies, and -# trim=true would eat the blank line between releases. -body = """## [{{ version | trim_start_matches(pat="v") }}](https://github.com/glim-sh/cuttle/releases/tag/{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }} -{% for group, commits in commits | group_by(attribute="group") %} -### {{ group | upper_first }} -{% for commit in commits %}{% if commit.scope %}- **{{ commit.scope }}:** {% else %}- {% endif %}{% if commit.breaking %}[**breaking**] {% endif %}{{ commit.message }} ([{{ commit.id | truncate(length=7, end="") }}](https://github.com/glim-sh/cuttle/commit/{{ commit.id }})) -{% endfor %}{% endfor %}{% if previous.version %} -**Full Changelog**: https://github.com/glim-sh/cuttle/compare/{{ previous.version }}...{{ version }} - -{% endif %}""" -trim = false - -[git] -conventional_commits = true -# Catch-all parser routes unconventional commits to Other instead of dropping them. -filter_unconventional = false -protect_breaking_commits = true -tag_pattern = "v[0-9].*" -commit_preprocessors = [ - { pattern = '\(#([0-9]+)\)', replace = "([#${1}](https://github.com/glim-sh/cuttle/pull/${1}))" }, -] -commit_parsers = [ - { message = "^chore: release", skip = true }, - { message = "^chore\\(release\\)", skip = true }, - { message = "^chore\\(deps\\)", skip = true }, - { message = "^[a-z]+(\\(.+\\))?!:", group = "๐Ÿ›  Breaking Changes" }, - { body = "^BREAKING CHANGE", group = "๐Ÿ›  Breaking Changes" }, - { message = "^feat", group = "๐ŸŽ‰ New Features" }, - { message = "^fix", group = "๐Ÿ› Bug Fixes" }, - { message = "^perf", group = "๐Ÿš€ Performance" }, - { message = "^refactor", group = "๐Ÿšœ Refactor" }, - { message = "^docs", group = "๐Ÿ“š Documentation" }, - { message = "^chore|^ci|^build|^test", group = "๐Ÿงน Chores" }, - { message = "^.*", group = "๐Ÿ”ง Other" }, -] diff --git a/.github/release-please-config.json b/.github/release-please-config.json index 159d8b2..0af7bed 100644 --- a/.github/release-please-config.json +++ b/.github/release-please-config.json @@ -1,14 +1,15 @@ { "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", - "release-type": "python", + "release-type": "go", "include-component-in-tag": false, "bump-minor-pre-major": true, "bump-patch-for-minor-pre-major": true, "packages": { ".": { - "package-name": "cuttle-browser", - "skip-changelog": true, - "extra-files": [{ "type": "generic", "path": "SKILL.md" }] + "package-name": "cuttle", + "extra-files": [ + "internal/cli/SKILL.md" + ] } } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90596d1..ff74597 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,30 +1,157 @@ name: ci -# Lint + type-check the authored Python (the `just check` gate, in check-only -# mode - CI never mutates). Runs on human PRs and pushes to main. NOTE: the -# release PR is opened by github-actions via github.token, which GitHub does not -# let trigger workflows, so this does not run on it - that PR only bumps version -# strings and needs no code CI. Heavier gates (`just smoke`, real-amd64) stay -# manual pre-merge; see AGENTS.md. on: - pull_request: push: branches: [main] + pull_request: +# Read-only by default; the release job elevates for its own publishing. permissions: contents: read -concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true +# Concurrency is per-job, not workflow-level: a build stays cancellable, but the +# release job must never be cancelled mid-publish. jobs: - check: + build-and-test: + runs-on: ubuntu-latest + concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - uses: docker/setup-buildx-action@v4 + + - name: Lint + uses: golangci/golangci-lint-action@v9 + with: + version: v2.12 + + - name: Test + run: | + go install gotest.tools/gotestsum@latest + gotestsum --format github-actions -- -race -coverprofile=coverage.out -covermode=atomic ./... + + - name: Vuln + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + + # helm is preinstalled on ubuntu-latest; guard the chart against drift. + - name: Helm lint + run: | + helm lint ops/helm/cuttle + helm template ops/helm/cuttle >/dev/null + + # Build the amd64 image and smoke it over raw CDP (per-seed isolation, + # stealth coherence, WebGL via ANGLE). buildx GHA cache reuses the apt + + # clark/clearcote + KasmVNC layers so only a Chrome bump re-downloads them. + - name: Build image (cached) + uses: docker/build-push-action@v7 + with: + context: . + file: ops/docker/Dockerfile + platforms: linux/amd64 + load: true + tags: cuttle:ci + cache-from: type=gha + cache-to: type=gha,mode=max,ignore-error=true + + - name: Smoke (Go harness over CDP) + run: | + set -euo pipefail + docker run -d --name cuttle-smoke -p 127.0.0.1:19222:9222 --shm-size=2g cuttle:ci + for _ in $(seq 1 60); do + curl -sf http://127.0.0.1:19222/json/version >/dev/null 2>&1 && break + sleep 1 + done + CUTTLE_URL=http://127.0.0.1:19222 go run ./test/smoke + + - name: Teardown + if: always() + run: docker rm -f cuttle-smoke >/dev/null 2>&1 || true + + # Runs only on push to main, and only after build-and-test is green. + # release-please maintains the release PR; merging it cuts the tag + GitHub + # release, and the gated steps then publish (GoReleaser binaries + Homebrew + # cask, and the GHCR image) in this same run - no PAT-triggered tag handoff. + release: + needs: build-and-test + if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest + permissions: + contents: write # tag + release, GoReleaser upload + pull-requests: write # release-please PR + packages: write # push image to GHCR + concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false steps: + # release-please-action is API-driven and needs no checkout. + - uses: googleapis/release-please-action@v5 + id: rp + with: + config-file: .github/release-please-config.json + manifest-file: .github/.release-please-manifest.json + + # Everything below runs only when merging the release PR actually cut a + # release (release_created == true). - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.3.2 - - run: uv run ty check - - run: uv run ruff check --output-format github - - run: uv run ruff format --check - - run: uv run python scripts/check-skill-version.py + if: steps.rp.outputs.release_created == 'true' + with: + fetch-depth: 0 # GoReleaser needs full history + the new tag + + - uses: actions/setup-go@v7 + if: steps.rp.outputs.release_created == 'true' + with: + go-version-file: go.mod + + - uses: goreleaser/goreleaser-action@v7 + if: steps.rp.outputs.release_created == 'true' + with: + version: "~> v2" + args: release --clean --config ops/config/goreleaser.yaml + env: + GITHUB_TOKEN: ${{ github.token }} + # Push the Homebrew cask to the separate tap repo. + GH_RELEASE_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }} + + - uses: docker/setup-buildx-action@v4 + if: steps.rp.outputs.release_created == 'true' + + - name: Log in to GHCR + if: steps.rp.outputs.release_created == 'true' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image metadata + id: meta + if: steps.rp.outputs.release_created == 'true' + uses: docker/metadata-action@v6 + with: + images: ghcr.io/glim-sh/cuttle + tags: | + type=raw,value=${{ steps.rp.outputs.version }} + type=raw,value=${{ steps.rp.outputs.major }}.${{ steps.rp.outputs.minor }} + type=raw,value=latest + type=sha + + # clark/clearcote ship linux-x64 prebuilts only, so the image is amd64-only. + - name: Build and push image + if: steps.rp.outputs.release_created == 'true' + uses: docker/build-push-action@v7 + with: + context: . + file: ops/docker/Dockerfile + platforms: linux/amd64 + push: true + build-args: | + VERSION=${{ steps.rp.outputs.version }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index b8b9612..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: release - -# PR-merge-driven release (release-please), single workflow like pond. -# -# On every push to main, release-please maintains a `chore(main): release X.Y.Z` -# PR: it derives the version from conventional commits, bumps pyproject.toml + -# SKILL.md's frontmatter version, and (on the PR) nothing else - the changelog -# stays git-cliff's (skip-changelog in .github/release-please-config.json). -# -# Merging that PR is the release: release-please tags vX.Y.Z + opens the GitHub -# release, then the gated jobs below (release_created == true) publish - sdist/ -# wheel -> PyPI (OIDC), image -> GHCR, git-cliff notes onto the release + dist -# assets, CHANGELOG.md/uv.lock synced back to main, and the homebrew formula. -# Everything runs in THIS workflow run, so no PAT-triggered tag handoff is -# needed. No test gate: the manual harness is the compat gate, run before merge. -# See docs/RELEASING.md. -on: - push: - branches: [main] - -permissions: - contents: read - -# Never cancel a release mid-publish; a newer push queues behind it. -concurrency: - group: release-${{ github.ref }} - cancel-in-progress: false - -jobs: - release-please: - runs-on: ubuntu-latest - permissions: - contents: write # create the tag + GitHub release on merge - pull-requests: write # open/update the release PR - outputs: - release_created: ${{ steps.rp.outputs.release_created }} - tag_name: ${{ steps.rp.outputs.tag_name }} - version: ${{ steps.rp.outputs.version }} - major: ${{ steps.rp.outputs.major }} - minor: ${{ steps.rp.outputs.minor }} - steps: - # Default token (github.token) is enough: everything publishes in this same - # run, so we never rely on the created tag to trigger another workflow. - - uses: googleapis/release-please-action@v5 - id: rp - with: - config-file: .github/release-please-config.json - manifest-file: .github/.release-please-manifest.json - - dist: - needs: release-please - if: ${{ needs.release-please.outputs.release_created == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.3.2 - - run: uv build - - uses: actions/upload-artifact@v7 - with: - name: dist - path: dist/ - - pypi: - needs: [release-please, dist] - if: ${{ needs.release-please.outputs.release_created == 'true' }} - runs-on: ubuntu-latest - environment: pypi - permissions: - id-token: write # PyPI trusted publishing (OIDC) - no token secret - steps: - - uses: astral-sh/setup-uv@v8.3.2 - - uses: actions/download-artifact@v8 - with: - name: dist - path: dist - - run: uv publish dist/* - - image: - needs: release-please - if: ${{ needs.release-please.outputs.release_created == 'true' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@v7 - - - uses: docker/setup-buildx-action@v4 - - - name: Log in to GHCR - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # push-to-main trigger has no git tag ref, so feed the version explicitly - # (type=semver reads the ref). `latest` is declared, not inherited from - # metadata-action's implicit latest=auto - the CLI falls back to it. - - name: Metadata - id: meta - uses: docker/metadata-action@v6 - with: - images: ghcr.io/glim-sh/cuttle - tags: | - type=raw,value=${{ needs.release-please.outputs.version }} - type=raw,value=${{ needs.release-please.outputs.major }}.${{ needs.release-please.outputs.minor }} - type=raw,value=latest - type=sha - - # clark/clearcote ship linux-x64 prebuilts only, so the image is amd64-only. - - name: Build and push - uses: docker/build-push-action@v7 - with: - context: . - platforms: linux/amd64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - # release-please already created the tag + release; here we put git-cliff notes - # + dist assets on it, and sync the git-cliff CHANGELOG.md and uv.lock (which - # release-please does not touch) back to main. - finalize: - needs: [release-please, dist] - if: ${{ needs.release-please.outputs.release_created == 'true' }} - runs-on: ubuntu-latest - permissions: - contents: write # gh release edit + commit CHANGELOG/uv.lock back to main - steps: - - uses: actions/checkout@v7 - with: - ref: main - fetch-depth: 0 # git-cliff walks tags for --latest and the compare link - - uses: astral-sh/setup-uv@v8.3.2 - - uses: actions/download-artifact@v8 - with: - name: dist - path: dist - # git-cliff ships on PyPI, so uvx runs the same binary as local git-cliff. - # The leading `## [version]` heading is stripped - GitHub shows the tag as - # the title; the Full Changelog footer survives (pond-style). - - name: Release notes + assets - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ needs.release-please.outputs.tag_name }} - run: | - set -euo pipefail - uvx git-cliff --config .github/cliff.toml --latest --strip header | awk '!s && /^## /{s=1; next} s' > "$RUNNER_TEMP/notes.md" - gh release edit "$TAG" --notes-file "$RUNNER_TEMP/notes.md" \ - || gh release create "$TAG" --notes-file "$RUNNER_TEMP/notes.md" --verify-tag - gh release upload "$TAG" dist/* --clobber - - name: Sync CHANGELOG.md + uv.lock back to main - env: - TAG: ${{ needs.release-please.outputs.tag_name }} - run: | - set -euo pipefail - uvx git-cliff --config .github/cliff.toml -o CHANGELOG.md - uv lock - if ! git diff --quiet CHANGELOG.md uv.lock; then - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add CHANGELOG.md uv.lock - git commit -m "chore(release): sync CHANGELOG + uv.lock for $TAG [skip ci]" - git push origin HEAD:main - fi - - # The formula fetches the sdist from the GitHub release, so it publishes only - # after finalize has uploaded the dist assets to that release. - homebrew: - needs: [release-please, finalize] - if: ${{ needs.release-please.outputs.release_created == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/download-artifact@v8 - with: - name: dist - path: dist - - name: Render and push formula - env: - GH_RELEASE_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }} - V: ${{ needs.release-please.outputs.version }} - run: | - set -euo pipefail - python3 scripts/render-homebrew-formula.py \ - --version "$V" --sdist "dist/cuttle_browser-$V.tar.gz" --out "$RUNNER_TEMP/cuttle.rb" - tap="$RUNNER_TEMP/tap" - rm -rf "$tap" - git clone --depth 1 "https://x-access-token:$GH_RELEASE_TOKEN@github.com/tenequm/homebrew-tap" "$tap" - cp "$RUNNER_TEMP/cuttle.rb" "$tap/Formula/cuttle.rb" - git -C "$tap" add Formula/cuttle.rb - git -C "$tap" diff --quiet --staged || git -C "$tap" -c user.name=ci -c user.email=ci@cuttle commit -m "cuttle $V" - git -C "$tap" push diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml deleted file mode 100644 index 089bda7..0000000 --- a/.github/workflows/smoke.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: smoke - -# The `just smoke` gate in CI: build the amd64 image and run test/harness.py -# (raw CDP over websockets - per-seed fingerprint isolation, stealth coherence, -# connection stability) against a throwaway container. Free on public-repo -# runners, but heavy (clark+clearcote are ~335MB of downloads), so it is -# path-filtered to image-affecting changes only and uses buildx GHA cache to -# reuse the browser/apt layers across runs. CLI/docs-only PRs skip it. This is -# NOT the full compatibility gate - that is the real-amd64 live-site run. -on: - pull_request: - paths: - - "Dockerfile" - - "bin/**" - - "vendor/**" - - "test/**" - - "scripts/rename-fonts.py" - - "uv.lock" - - "pyproject.toml" - - ".github/workflows/smoke.yml" - push: - branches: [main] - paths: - - "Dockerfile" - - "bin/**" - - "vendor/**" - - "test/**" - - "scripts/rename-fonts.py" - - "uv.lock" - - "pyproject.toml" - - ".github/workflows/smoke.yml" - -permissions: - contents: read - -concurrency: - group: smoke-${{ github.ref }} - cancel-in-progress: true - -jobs: - smoke: - # Skip release-please's version-bump PRs: they only touch pyproject/uv.lock - # version strings (which match the paths filter) but never change the image. - if: ${{ !startsWith(github.head_ref, 'release-please--') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: docker/setup-buildx-action@v4 - - uses: astral-sh/setup-uv@v8.3.2 - - # amd64 image, loaded into the local docker so we can run it. buildx GHA - # cache reuses the apt + clark/clearcote + KasmVNC layers across runs, so - # only a Chrome bump (changed pinned tags) re-downloads the 335MB forks. - - name: Build image (cached) - uses: docker/build-push-action@v7 - with: - context: . - platforms: linux/amd64 - load: true - tags: cuttle:ci - cache-from: type=gha - # Cache is an optimization; a transient GHA-cache reservation flake must - # not fail a build that otherwise succeeded (ignore-error). - cache-to: type=gha,mode=max,ignore-error=true - - - name: Smoke (harness over CDP) - run: | - set -euo pipefail - docker run -d --name cuttle-smoke -p 127.0.0.1:19222:9222 --shm-size=2g cuttle:ci - for _ in $(seq 1 60); do - curl -sf http://127.0.0.1:19222/json/version >/dev/null 2>&1 && break - sleep 1 - done - CUTTLE_URL=http://127.0.0.1:19222 uv run python test/harness.py - - - name: Teardown - if: always() - run: docker rm -f cuttle-smoke >/dev/null 2>&1 || true diff --git a/.gitignore b/.gitignore index 05f453c..3bbf78f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ -__pycache__/ -*.pyc -*.egg-info/ -build/ -dist/ -.venv/ -venv/ +# Go build artifacts +/cuttle +/coverage.out +*.test + +# GoReleaser output +/dist/ + +# local .DS_Store .env CLAUDE.md diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..96dc06c --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,147 @@ +version: "2" + +run: + timeout: 5m + +linters: + default: standard + enable: + - bodyclose + - copyloopvar + - dupl + - durationcheck + - err113 + - errname + - errorlint + - exhaustive + - exptostd + - fatcontext + - goconst + - gocritic + - gosec + - intrange + - misspell + - modernize + - musttag + - nakedret + - nestif + - nilerr + - noctx + - nolintlint + - nonamedreturns + - perfsprint + - prealloc + - revive + - sqlclosecheck + - testifylint + - thelper + - unconvert + - unparam + - usestdlibvars + - usetesting + - wastedassign + - whitespace + - wrapcheck + settings: + govet: + enable: + - shadow + gocritic: + enabled-checks: + - nestingReduce + goconst: + # Table-driven tests legitimately repeat literals; only production + # repetition is worth hoisting to a constant. + ignore-tests: true + wrapcheck: + # Errors returned from our own internal packages are already domain errors; + # re-wrapping them at every hop adds noise, not context. Only external + # (stdlib / third-party) errors must be wrapped. + ignore-package-globs: + - github.com/glim-sh/cuttle/* + revive: + enable-all-rules: true + rules: + # Writes to stdout/stderr and to an in-memory strings.Builder never + # return an error worth handling. + - name: unhandled-error + arguments: + - "fmt\\.Fprint.*" + - "fmt\\.Print.*" + - "strings\\.Builder\\.Write.*" + - "bytes\\.Buffer\\.Write.*" + # The comma-ok form (x, _ := v.(T)) is a deliberate best-effort read. + - name: unchecked-type-assertion + arguments: + - acceptIgnoredAssertionResult: true + # Style rules that conflict with idiomatic Go / gofumpt or add noise + # without catching defects. Everything else stays on. + - name: line-length-limit + disabled: true + - name: add-constant + disabled: true + - name: cognitive-complexity + disabled: true + - name: cyclomatic + disabled: true + - name: function-length + disabled: true + - name: argument-limit + disabled: true + - name: enforce-switch-style + disabled: true + - name: max-public-structs + disabled: true + - name: flag-parameter + disabled: true + - name: unused-receiver + disabled: true + # Conflicts with the nonamedreturns linter (which forbids named returns). + - name: confusing-results + disabled: true + - name: function-result-limit + disabled: true + # Anonymous structs are idiomatic for JSON fixtures. + - name: nested-structs + disabled: true + errcheck: + check-type-assertions: true + exclusions: + generated: strict + presets: + - comments + - std-error-handling + - common-false-positives + rules: + - path: _test\.go + linters: + - gocyclo + - errcheck + - dupl + - gosec + - wrapcheck + - goconst + - revive + - bodyclose + - noctx + +formatters: + enable: + - gofumpt + - goimports + settings: + gofumpt: + extra-rules: true + exclusions: + generated: strict + +output: + formats: + text: + path: stdout + print-linter-name: true + colors: true + sort-order: + - linter + - file + show-stats: true diff --git a/AGENTS.md b/AGENTS.md index 386fd53..8a0cf02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,132 +1,39 @@ -# AGENTS.md - working in cuttle - -cuttle is a stealth-Chromium CDP farm: a patched CDP multiplexer (`bin/cuttleserve`) -that spawns one stealth Chrome per fingerprint seed, with a free redistributable -fork binary (clark/clearcote) baked into the image. The real deliverable is the -Docker image (`ghcr.io/glim-sh/cuttle`). Read `README.md` for the product view. - -## The one rule that shapes everything: the vendored Python lives under `vendor/` - -- `vendor/cloakbrowser/` (`config.py`, `geoip.py` verbatim; `browser.py` trimmed) - and `bin/cuttleserve` (patched `cloakserve`) are vendored/patched from upstream - CloakHQ `cloakbrowser`. **Do NOT reformat, re-lint, re-type, or restyle them.** - Reformatting breaks the "verbatim" provenance and blows up `scripts/sync.sh` diffs. - `ruff`/`ty` are scoped to exclude `vendor/`; keep it that way. The package imports - as `cloakbrowser` (via `[tool.setuptools.package-dir]`) so cuttleserve's imports - and the sync diff stay upstream-verbatim. -- To update the vendored subset: `just vendor-sync` fetches the pinned upstream - (see `docs/UPSTREAM.md`) and prints a reviewable diff; re-apply the trims/patches - by hand, then bump the pinned ref. Never blind-copy over the vendored files. -- Authored code you own and should keep clean: the `cuttle/` package (host CLI + - experimental CDP-screencast viewer), `test/harness.py`, `scripts/rename-fonts.py`, - the `Dockerfile`, docs, and the small glue in `cuttleserve` (the proxy-auth - injection + the `_stamp_sw_context` service_worker stamp - treat those two - patches as load-bearing; they are why the fork works). - -## Dev stack (uv + ruff + ty + just) - -```bash -uv sync # create/refresh .venv from uv.lock (CLI deps + dev tools) -uv sync --group server # + the container-only deps, to run bin/cuttleserve bare-metal -just check # ty check + ruff check --fix + ruff format (authored code only) -just build # docker buildx build --platform linux/amd64 -> cuttle:local -just smoke # build, run a throwaway container, run test/harness.py against it -just vendor-sync # show upstream drift of the vendored subset -just release-preview # preview the changelog the next release-please PR will carry -uv add # add a runtime dep (updates pyproject + uv.lock) -``` - -- **Deps are split by consumer.** `[project.dependencies]` (aiohttp, websockets) is - what the published CLI needs - it is all that pip/brew/nix install. The - container-only `server` group (httpx, geoip2, socksio) is what `bin/cuttleserve` - needs via `cloakbrowser.geoip`, which imports them lazily; only the Dockerfile - installs it. Adding a dep to `[project.dependencies]` that only cuttleserve uses - puts it in the brew formula and every user's install - put it in `server` instead. -- **`cuttle up` defaults to the published image for its own version**, not your - local build: to drive a local image, `just build` then `cuttle up --image cuttle:local`. -- Python 3.12 (matches the base image + vendored code). `uv.lock` is committed and - pins the image's Python deps - the Dockerfile installs from it (reproducible builds). -- The image is **linux/amd64 only**: clark/clearcote ship linux-x64 prebuilts. On an - arm64 host the build/run is emulated (fine for local dev + a smoke; ~2s page - loads). Production runs it native on an amd64 server. - -## Validation model (two layers, different jobs) - -- `test/harness.py` - fast, local, client-agnostic (raw CDP over `websockets`). - Checks per-seed fingerprint isolation, stealth coherence, connection stability. - It CANNOT observe the playwright-core service_worker crash (only a playwright client - can) and does not clear real challenges. Run it on any bump. -- **Real amd64 deployment** - runs the actual playwright-core consumer path against - live sites. This is the gate that surfaces a new playwright-crashing CDP quirk and - confirms real challenge clears. See `docs/UPGRADE.md`. Keep this validation OUT of - this public repo (it names real sites); it lives with the consumer. - -## Bind host (don't regress this) - -`cuttleserve` serves CDP on `0.0.0.0:9222` inside a container and `127.0.0.1` on -bare metal (loopback-only there, for safety). "Inside a container" is detected for -docker, podman, AND k8s/containerd - via `/.dockerenv`, `/run/.containerenv`, -`KUBERNETES_SERVICE_HOST`, or a container cgroup - and `CUTTLESERVE_HOST` overrides. -Do NOT gate the bind on `/.dockerenv`/`/run/.containerenv` alone: under containerd -BOTH markers are absent, which silently pins the listener to loopback and refuses -every cross-container client. Docker Desktop has `/.dockerenv`, so a Docker-only -setup hides the bug - the real-amd64 gate (cross-container) is what catches it. This -is why no `touch /run/.containerenv` startup hack is needed anywhere. - -## Bumping Chrome - -Update the pinned `CLARK_*` / `CLEARCOTE_*` build args in the `Dockerfile`, rebuild, -`just smoke`, then run the real-amd64 gate. `docs/UPGRADE.md` has the runbook; -`docs/BUILD-FROM-SOURCE.md` is break-glass only. - -## Releasing (release-please, PR-merge-driven) - -Releases are driven by release-please (`.github/release-please-config.json`), NOT a local -command. Never hand-craft a release PR or pick a version by hand - release-please -derives both. Full runbook: `docs/RELEASING.md`. - -- **Flow.** Land conventional commits on `main` -> release-please auto-opens/updates - one `chore(main): release X.Y.Z` PR (bumping `pyproject.toml` + `SKILL.md`'s - frontmatter version) -> run the gates (`just smoke` + the real-amd64 check) -> - **merge that PR**. The merge IS the release: the `release.yml` run then publishes - PyPI/GHCR/GitHub release/homebrew and syncs `CHANGELOG.md` + `uv.lock` back to - `main`. A feature PR is never "the release PR". -- **What opens a release PR.** Only `feat`, `fix`, `perf`, `revert` (or any - breaking-marked commit) are user-facing enough to trigger one. A batch of only - `chore`/`docs`/`refactor`/`ci`/`test`/`build`/`style` does NOT open a release PR - - land a `feat`/`fix` too, or force it (below). -- **Version bump (pre-1.0, `bump-*-pre-major` enabled in the config).** Every - non-breaking commit -> a **patch** bump (`0.3.0` -> `0.3.1`); a breaking marker -> - a **minor** bump (`0.3.0` -> `0.4.0`), never a 1.0.0 major. -- **Breaking markers count on ANY type - the key difference from pond/release-plz.** - release-please reads "breaking" off the message, type-agnostically: a `!` header - (`feat!:` AND `docs!:`, `chore!:`, `refactor!:`) or a `BREAKING CHANGE:` footer - ALL count, on any type, even on an empty commit. pond's release-plz only honors - breaking on `feat!`/`fix!` with a real diff - do NOT carry that assumption here. - Put `!`/`BREAKING CHANGE:` only on the commit you actually mean to cut a minor for. -- **Force a version.** Add `Release-As: X.Y.Z` (case-insensitive) to a commit body - on `main` before merging - any commit, incl. an empty - `git commit --allow-empty -m "chore: release X.Y.Z" -m "Release-As: X.Y.Z"`; - newest wins. Use for a deliberate bump the commit types would not derive (0.3.0 - was forced this way over the natural 0.2.1). -- **Changelog is git-cliff, from commit subjects** (`.github/cliff.toml`; `skip-changelog` - keeps release-please out of it). Nothing to hand-edit on the release PR - a clear, - scoped commit subject IS the changelog line. `CHANGELOG.md` and the GitHub release - body are both git-cliff output; the emoji section taxonomy is owned by - `.github/cliff.toml`'s `commit_parsers` (ported from pond) - do not restyle it. -- **Prereq (one-time, already enabled).** The `glim-sh` org + `cuttle` repo both - allow "GitHub Actions to create and approve pull requests"; without it - release-please cannot open the PR. It runs on the default `github.token`. +# AGENTS.md - cuttle + +cuttle is a stealth-Chromium CDP farm: `cuttle serve` is a CDP multiplexer that +spawns one stealth Chrome per fingerprint seed, routing per-seed identity +(fingerprint, proxy, geoip) over CDP. A single static Go binary; the daemon runs +in a Python-free container. + +## Layout + +- `cmd/cuttle/` - CLI entrypoint. `internal/` - the packages (serve daemon, + fingerprint arg-builder, backends, profile store, cdp, config, mcp). Go 1.26, + gofumpt, golangci-lint v2, just. Module: `github.com/glim-sh/cuttle`. +- `ops/docker/` - the container build assets: `Dockerfile` (stealth-Chromium + runtime, clark/clearcote forks + headed Xvfb/openbox + KasmVNC, linux/amd64 + only), `bin/` (entrypoint + VNC viewer), and `winfonts/` (pre-baked + metric-compatible free fonts reporting Windows family names; see its README). + Build context is the repo root: `just build-image` (or `docker build -f + ops/docker/Dockerfile .`). +- `test/smoke/` - neutral, self-contained CDP smoke harness (`go run + ./test/smoke` against a running container). +- `ops/helm/cuttle/` - Helm chart for the k8s backend. `docs/` - plans + docs. ## Non-negotiables -- **This is a PUBLIC repo.** Never add: internal infra references (clusters, k8s, - namespaces, proxies, secrets), named commercial scraping targets or "bypass X on - " framing, or any credential. The harness stays neutral and self-contained. -- **No proprietary binary.** cuttle uses only free forks (clark MIT, clearcote BSD-3) - and the MIT `cloakserve`/`cloakbrowser` subset. Keep `LICENSE` + `THIRD-PARTY.md` - accurate; never bake a CloakBrowser binary or its base image into any layer. -- **No license/Pro code.** The vendored subset deliberately excludes CloakBrowser's - license/widevine/launch machinery. Do not reintroduce it. -- `CLOAKBROWSER_BINARY_PATH` selects the engine (`/opt/clark/chrome` default, - `/opt/clearcote/chrome` fallback) - kept as the fork's own override contract. +- This is a PUBLIC repo. Never add internal infra references (clusters, k8s + namespaces, proxies, secrets), named commercial scraping targets or + "bypass X on " framing, or any credential. +- No proprietary binaries: only the free stealth-Chromium forks (clark MIT, + clearcote BSD-3). The daemon and fingerprint code are authored Go, not vendored + from any licensed browser product. +- Stealth output is the whole game: fingerprint arg-building, proxy + normalization, and geoip are snapshotted in the golden + `internal/fingerprint/testdata/golden.json` (regenerate with `just + parity-golden`). The golden is a regression tripwire - it turns any change to + that output into a diff someone must consciously regenerate and review, so a + stealth drift can never land silently. (It was originally captured + byte-for-byte from the now-removed Python oracle.) +- Conventional Commits (`type(scope): description`); releases are + release-please-driven from `main`, built and published by GoReleaser. diff --git a/Justfile b/Justfile index 6fac233..72a1562 100644 --- a/Justfile +++ b/Justfile @@ -1,43 +1,97 @@ -# cuttle - stealth-Chromium CDP farm +set shell := ["bash", "-euo", "pipefail", "-c"] -image := "cuttle:local" +binary := "cuttle" -# List recipes +[private] default: - @just --list + @just --list --unsorted -# Type-check + lint + format the authored Python (test/ + scripts/) -check: - uv run ty check - uv run ruff check --fix - uv run ruff format - uv run python scripts/check-skill-version.py +# โ”€โ”€ Code Quality โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -# Build the linux/amd64 image (clark/clearcote are linux-x64 only) +# Format all Go code +[group('quality')] +fmt: + golangci-lint fmt ./... + +# Check formatting without modifying (CI-safe) +[group('quality')] +fmt-check: + gofumpt -d . 2>&1 | (! grep -q '^') || (gofumpt -l . && exit 1) + +# Run linter +[group('quality')] +lint: + golangci-lint run ./... + +# Run linter with auto-fix +[group('quality')] +lint-fix: + golangci-lint run --fix ./... + +# Run vulnerability check +[group('quality')] +vuln: + govulncheck ./... + +# โ”€โ”€ Testing โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# Run all tests with race detection +[group('test')] +test *args="./...": + gotestsum --format testname -- -race {{ args }} + +# Run tests with coverage +[group('test')] +test-cov: + gotestsum --format testname -- -race -coverprofile=coverage.out -covermode=atomic ./... + go tool cover -func=coverage.out + +# โ”€โ”€ Build โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# Build the binary +[group('build')] build: - docker buildx build --platform linux/amd64 --load -t {{image}} . - -# Build, then run the neutral smoke harness against a throwaway container -smoke: build - #!/usr/bin/env bash - set -euo pipefail - docker rm -f cuttle-smoke >/dev/null 2>&1 || true - docker run -d --name cuttle-smoke -p 127.0.0.1:19222:9222 --shm-size=2g {{image}} >/dev/null - trap 'docker rm -f cuttle-smoke >/dev/null 2>&1 || true' EXIT - for _ in $(seq 1 60); do curl -sf http://127.0.0.1:19222/json/version >/dev/null 2>&1 && break; sleep 1; done - CUTTLE_URL=http://127.0.0.1:19222 uv run python test/harness.py - -# Show drift of the vendored upstream subset (reviewable diff; does not overwrite) -vendor-sync: - ./scripts/sync.sh - -# Releases are PR-merge-driven via release-please (see docs/RELEASING.md): land -# conventional commits on main, run the gates, then merge the release PR it -# opens. This recipe just previews the changelog the next release would carry. -release-preview: - uvx git-cliff --config .github/cliff.toml --unreleased --tag "$(uvx git-cliff --config .github/cliff.toml --bumped-version)" - -# Remove build artifacts + go build -o {{ binary }} ./cmd/{{ binary }} + +# Build optimized release binary +[group('build')] +build-release: + CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o {{ binary }} ./cmd/{{ binary }} + +# Build the container image (amd64; clark/clearcote ship linux-x64 prebuilts only) +# BuildKit is required so the per-Dockerfile ops/docker/Dockerfile.dockerignore is +# honored (there is no root .dockerignore); classic builder would send the whole repo. +[group('build')] +build-image tag="cuttle:local": + DOCKER_BUILDKIT=1 docker build --platform linux/amd64 -f ops/docker/Dockerfile -t {{ tag }} . + +# โ”€โ”€ Dependencies โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# Tidy and verify modules +[group('deps')] +tidy: + go mod tidy + go mod verify + +# โ”€โ”€ CI โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# Full CI gate (format check + lint + test) +[group('ci')] +check: fmt-check lint test + @echo "All checks passed" + +# Regenerate the fingerprint parity golden snapshot from the Go primitives +[group('ci')] +parity-golden: + GOTOOLCHAIN=auto go test ./internal/fingerprint -run TestGolden -update + +# Validate the GoReleaser config (lives under ops/config, not repo root) +[group('ci')] +release-check: + goreleaser check --config ops/config/goreleaser.yaml + +# Clean build artifacts +[group('ci')] clean: - rm -rf build dist *.egg-info .ruff_cache - find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + go clean + rm -f {{ binary }} coverage.out diff --git a/README.md b/README.md index c22603a..8f6764a 100644 --- a/README.md +++ b/README.md @@ -1,140 +1,160 @@ # cuttle -A stealth-Chromium CDP farm. `cuttle` runs a patched Chrome DevTools Protocol -(CDP) multiplexer that spawns one stealth Chrome per fingerprint seed, giving -each seed its own coherent browser identity - fingerprint, proxy, geoip, locale, -and timezone - behind a single CDP endpoint. Point any CDP client -(Playwright, Puppeteer, `chromium.connectOverCDP`) at it and select a seed with -a query parameter. +`cuttle` - a stealth-Chromium CDP farm and a universal +browser-lifecycle CLI. It runs a patched Chrome DevTools Protocol (CDP) +multiplexer that spawns one stealth Chrome per fingerprint seed - each with its +own coherent identity (fingerprint, proxy, geoip, locale, timezone) behind a +single CDP endpoint - and manages that browser wherever you want it: locally in +Docker, in a Kubernetes cluster, over SSH, or against a pre-exposed URL. The Chrome engine is a free, redistributable stealth-Chromium fork baked into the image - [clark](https://github.com/clark-labs-inc/clark-browser) (MIT) by -default, [clearcote](https://github.com/clearcotelabs/clearcote-browser) -(BSD-3) as a fallback - so there is no proprietary binary and no license to -manage. The multiplexer is a patched derivative of CloakHQ's MIT-licensed -[`cloakserve`](https://github.com/CloakHQ/cloakbrowser). +default, [clearcote](https://github.com/clearcotelabs/clearcote-browser) (BSD-3) +as a fallback. No proprietary binary. Maintained by [glim.sh](https://glim.sh). -Maintained by [glim.sh](https://glim.sh). +## Install / build -## Why +```bash +brew install tenequm/tap/cuttle # homebrew cask (macOS/Linux) +go install github.com/glim-sh/cuttle/cmd/cuttle@latest # from source (needs Go 1.26+) +just build # -> ./cuttle (native) +just build-release # CGO_ENABLED=0, -trimpath -ldflags='-s -w' +``` -The stealth-scraping stack normally reconciles several independently-drifting -upstreams by hand: the CDP multiplexer, the base image, and the Chrome fork -binary - each of which moves on its own schedule, and Chrome ships a new major -roughly every four weeks. cuttle owns the orchestration in one repo and consumes -the browser as a pinned prebuilt, turning "always check what still works" into -"pick a binary release, run the harness, ship or don't." One decision, one test. +The container image is `ghcr.io/glim-sh/cuttle`. The CLI shells out to Docker, +`kubectl`, `helm`, and `ssh` as the active context requires - it inherits your +existing kube context, ssh config, and routing with no cuttle-specific setup. -## Quickstart +## Quickstart (local Docker) ```bash -docker run --rm -p 9222:9222 ghcr.io/glim-sh/cuttle:latest +cuttle up # start the container + VNC viewer +cuttle login https://accounts.google.com # sign in once via the viewer +cuttle status # browser + CDP state +cuttle down # graceful stop, keeps the profile ``` -Then connect a CDP client and pass a fingerprint seed: +`cuttle up` is idempotent and profile-preserving; `--image cuttle:local` drives a +local build. Point any CDP client at the printed endpoint and select a seed: ``` http://127.0.0.1:9222?fingerprint=12345 http://127.0.0.1:9222?fingerprint=12345&timezone=America/New_York&locale=en-US ``` -Each distinct `fingerprint` seed gets its own isolated Chrome with a stable, -coherent identity. To route a seed through an authenticated residential proxy, -pass it on the connect URL; cuttle strips the inline credentials and answers the -proxy's `407` over CDP, so fork binaries that reject inline credentials still -work. +## Contexts and backends -The image runs **headed by default** (the default command is -`cuttleserve --headless=false`, on a built-in Xvfb): headed Chrome clears -escalated anti-bot challenges that headless cannot. Override the command only to -change flags or the port. +A **context** names where the browser runs. It is selected by +`--context` > `CUTTLE_CONTEXT` > `default_context` in the config file > +built-in `local`. Config lives at `$XDG_CONFIG_HOME/cuttle/config.toml`; list +contexts with `cuttle context ls`. -## CLI (daily driver + login handoff) +| Backend | Where the browser runs | Reach | +|----------|------------------------|-------| +| `local` | Docker on this machine | direct `127.0.0.1`, no tunnel | +| `k8s` | a Deployment (`helm upgrade --install ops/helm/cuttle`) | `kubectl port-forward` -> ephemeral local port | +| `ssh` | Docker on a remote host | `ssh -L` -> ephemeral local port | +| `direct` | a pre-exposed CDP/VNC URL | the config URL, used as-is | -For a persistent local browser you can watch and log into via VNC - then drive -over CDP - install the host CLI, published on PyPI as **`cuttle-browser`** (the -command it installs is **`cuttle`**): +Every CDP/VNC operation runs against `127.0.0.1:`, so the transport (docker +/ port-forward / ssh tunnel / direct) is invisible to the rest of the CLI. -```bash -brew install tenequm/tap/cuttle # homebrew (macOS/Linux) -uv tool install cuttle-browser # or: pipx install cuttle-browser -nix run github:glim-sh/cuttle # nix flake, builds from source at any rev -uvx --from cuttle-browser cuttle up # or one-off with no install -``` +```toml +default_context = "cluster" -```bash -cuttle up # start the container + VNC viewer -cuttle login https://accounts.google.com # sign in once via the viewer; the CDP session stays logged in -cuttle down # graceful stop, keeps the profile -cuttle skill # print the full agent usage guide +[context.local] +backend = "local" + +[context.cluster] +backend = "k8s" +namespace = "browser" +release = "cuttle" +node_selector = { "glim.sh/browser" = "true" } +proxy = "http://user:pass@proxy.example:8080" # applied at browser startup + +[context.box] +backend = "ssh" +host = "user@box.example" + +[context.edge] +backend = "direct" +cdp_url = "http://cuttle.example:9222" +vnc_url = "http://cuttle.example:6080" ``` -`cuttle up` is idempotent and profile-preserving (logins survive `down`/`up`). -The CLI shells out to Docker and defaults to the published image matching its own -version (`cuttle-browser 0.3.0` runs `ghcr.io/glim-sh/cuttle:0.3.0`), so the CLI -and cuttleserve never skew; override with `--image` (e.g. `--image cuttle:local` -after `just build`). See [SKILL.md](SKILL.md) (or `cuttle skill`) for the full -workflow. +The context `proxy` is a server-level default applied to every seed at startup; +geoip (timezone/locale/exit-IP) follows it automatically. A connection can still +override it per-request with `?proxy=`. -## Engine swap +- `cuttle status` / `cuttle login` open an ephemeral forward for the command. +- `cuttle connect` holds the forward open in the foreground and prints the driver + briefing (Ctrl-C to end) - use it for an interactive or agent session. -Both fork binaries are baked in and selected by `CLOAKBROWSER_BINARY_PATH`: +## Profiles (local-canonical auth state) -- `/opt/clark/chrome` - clark, Chrome 148 (default) -- `/opt/clearcote/chrome` - clearcote, Chrome 149 (fallback) +A named **profile** is a cuttle seed whose auth state lives on your machine at +`$XDG_DATA_HOME/cuttle/profiles//storage_state.json` (Playwright +storageState shape: cookies + per-origin localStorage). `--profile ` threads +through `login`, `connect`, and `mcp`, and appends `?fingerprint=` to every +attach URL. -```bash -docker run --rm -p 9222:9222 \ - -e CLOAKBROWSER_BINARY_PATH=/opt/clearcote/chrome \ - ghcr.io/glim-sh/cuttle:latest +```toml +[profile.linkedin] +storage = "local" # default: checkout/checkin over CDP, nothing persists remotely +[profile.bot] +storage = "remote" # durable on the browser host (autonomous / always-on) ``` -## Bumping Chrome +Local-canonical flow: at session start the profile's stored state is injected +into a freshly spawned remote seed over CDP; a periodic checkpoint and the final +check-in extract the updated state back to your machine. A single-writer lock +prevents a profile from being attached in two places at once. -Update the pinned `CLARK_*` / `CLEARCOTE_*` build args in the `Dockerfile`, -rebuild, and run the harness. See [docs/UPGRADE.md](docs/UPGRADE.md). Building -a binary from source is documented as break-glass only in -[docs/BUILD-FROM-SOURCE.md](docs/BUILD-FROM-SOURCE.md). +Honest caveat: state resides locally *at rest*, but during an active session the +live cookies are necessarily on the remote browser (it must hold them to act as +you). `storage = "remote"` skips checkout/checkin entirely for always-on use where +your machine is not present to inject state. -## Testing +## MCP (agent drivers) -`test/harness.py` is a neutral, self-contained smoke (raw CDP over `websockets`) -that drives a running cuttle and checks per-seed fingerprint isolation, stealth -coherence, and connection stability under cold-cycle load. Run it before -publishing any bump. End-to-end validation against live sites is done separately -against a real amd64 deployment. See [test/README.md](test/README.md). +```bash +cuttle mcp # default driver (browser-use), current context +cuttle mcp --profile linkedin # point the driver at the linkedin profile seed +``` -To confirm a running seed presents a coherent identity (WebGL GPU string, -WebRTC/WebGPU, `navigator.webdriver`) and to tell benign Chrome log noise from a -real problem, see [docs/STEALTH-VERIFICATION.md](docs/STEALTH-VERIFICATION.md). +`cuttle mcp [driver]` installs the CDP driver if absent and writes its MCP client +config (JSON) pointed at the active context's CDP endpoint, with the profile seed +appended as `?fingerprint=` (e.g. `BU_CDP_URL=http://127.0.0.1:9222?fingerprint=linkedin`). +The context proxy is already applied server-side. -## Architecture +## `cuttle serve` -- `bin/cuttleserve` - the patched CDP multiplexer. Per-seed Chrome pool, - transparent proxy-auth over CDP, a service_worker `browserContextId` stamp - (so CDP clients do not crash on service workers), and fork launch-parity flags. -- `cuttle/` - a trimmed MIT subset of the `cloakbrowser` wrapper: the CDP - argument-builders plus geoip/config helpers. No license, widevine, or - behavioral-automation code. -- `scripts/` - `rename-fonts.py` (Windows font pack builder: metric-compatible - free fonts renamed to Windows family names, so a Windows-claiming fingerprint - is coherent) and `sync.sh` (re-sync helper for the vendored upstream subset). -- `docs/` - the upgrade runbook, build-from-source break-glass, and - `UPSTREAM.md` (provenance of the vendored subset). +`cuttle serve` is the in-container daemon (the image entrypoint): the CDP +multiplexer itself. It binds `0.0.0.0:9222` inside a container (detected for +docker/podman/k8s) and `127.0.0.1` on bare metal, spawns one Chrome per +`?fingerprint=` seed, answers authenticated-proxy `407`s over CDP, and rewrites +the `webSocketDebuggerUrl` host to the request's Host header so it stays correct +behind a port-forward or ssh tunnel. `CUTTLESERVE_PROXY` sets a default proxy; +`CUTTLESERVE_HOST` overrides the bind host. -## Notes and limits +## Development + +```bash +just check # fmt-check + lint (golangci-lint v2) + test (gotestsum -race) +just build # ./cuttle +just vuln # govulncheck +``` -- The image is **linux/amd64 only**: the clark/clearcote prebuilts ship linux-x64 - binaries. On an Apple Silicon host it runs emulated (fine for local dev and the - smoke); production runs it native on an amd64 server. The Python multiplexer - itself is arch-agnostic. -- cuttle does not include a browser-automation client library - use any CDP - client. It is the farm, not the scraper. +Business logic lives in `internal/`; `cmd/cuttle` is a thin entrypoint. The +fingerprint arg-builder, proxy normalization, and geoip resolution are +parity-tested byte-for-byte against a committed golden +(`internal/fingerprint/testdata/golden.json`, regenerated with `just +parity-golden`). The Dockerfile is Python-free: a static Go binary plus the +verbatim clark/clearcote engine and KasmVNC/noVNC stages. ## Licensing -cuttle is MIT ([LICENSE](LICENSE)). It vendors and redistributes third-party -software under their own terms - CloakHQ's `cloakserve`/`cloakbrowser` (MIT), -clark (MIT), and clearcote (BSD-3). No proprietary CloakBrowser binary is used -or redistributed. Full attributions and license texts are in -[THIRD-PARTY.md](THIRD-PARTY.md). +MIT ([LICENSE](LICENSE)). The image redistributes the clark (MIT) and clearcote +(BSD-3) stealth-Chromium binaries; the fingerprint and serve code is authored +Go. No proprietary or licensed browser binary is used or redistributed. Full +notices and attributions in [docs/THIRD-PARTY.md](docs/THIRD-PARTY.md). diff --git a/SKILL.md b/SKILL.md deleted file mode 100644 index a83a830..0000000 --- a/SKILL.md +++ /dev/null @@ -1,289 +0,0 @@ ---- -name: cuttle -description: Run and drive cuttle - a local stealth-Chromium browser farm with persistent logins, anti-detect fingerprints, and a human-handoff viewer for captchas and Cloudflare. Use whenever the user says to use the browser, or asks to automate, scrape, test, or sign into a website, or names agent-browser, browser-use (bu, bu-cli), or playwright-cli. `cuttle up` prints the live briefing with installed drivers, exact CDP attach commands, and each driver's own docs command. Attach to cuttle's warm session - never launch a fresh browser or new profile. -metadata: - version: "0.3.0" # x-release-please-version - image: "ghcr.io/glim-sh/cuttle" -allowed-tools: Bash(cuttle:*) Bash(uv:*) Bash(just:*) Bash(docker:*) Bash(curl:*) Bash(agent-browser:*) Bash(browser-use:*) Bash(playwright-cli:*) ---- - -# cuttle: local stealth-browser CDP farm - -[cuttle](https://github.com/glim-sh/cuttle) runs a patched CDP multiplexer -(`cuttleserve`) that spawns one stealth Chrome per fingerprint seed - each with -its own coherent identity (fingerprint, proxy, geoip, locale, timezone) - behind -a single CDP endpoint. The engine is a free stealth-Chromium fork (clark MIT, -default; clearcote BSD-3, fallback); no proprietary binary. Point any CDP client -at it - agent-browser, browser-use, Playwright, `chromium.connectOverCDP`. - -The `cuttle` CLI wraps a Docker container; it does not automate pages itself - -cuttle is the farm, not the scraper. Two ways to use it: - -- **Daily driver / login handoff** - one persistent browser you watch and log - into via VNC, driven over CDP. Use the CLI: `cuttle up`. Start below. -- **Multi-seed farm** - many isolated identities behind one endpoint, no VNC. - Run the container and pick a seed with `?fingerprint=`. See [Multi-seed farm](#multi-seed-farm). - -## Setup - -Requires Docker (or OrbStack). The CLI is published on PyPI as -**`cuttle-browser`**; the command it installs is **`cuttle`**. - -```bash -brew install tenequm/tap/cuttle # homebrew (macOS/Linux) -uv tool install cuttle-browser # persistent global install -nix run github:glim-sh/cuttle # nix flake (builds from source at any rev) -# one-off, no install (note --from: the package is cuttle-browser, the command is cuttle): -uvx --from cuttle-browser cuttle up -# or: pipx install cuttle-browser / pip install cuttle-browser -``` - -If `cuttle` is not on PATH (`command not found`), do not install it globally on -your own - run every command through `uvx --from cuttle-browser cuttle ...`. - -Then, from any directory: - -```bash -cuttle up # start the container with the VNC viewer on; pulls the image if needed -``` - -`up` (and `status`) print **the briefing**: CDP + viewer URLs, cuttle's -version, which driver CLIs are installed - with the exact attach command and -the self-doc command for each - plus routing rules and install advisories for -missing drivers. The briefing is the live source of truth; follow it over any -cached knowledge, including this guide. - -`up` is idempotent and profile-preserving: a stopped container is **restarted** -(logins persist), not recreated. Default ports: CDP 9222, VNC 6080. - -### Picking ports (important) - -Every subcommand takes `--cdp-port` and `--vnc-port`. Use them when the defaults -are taken: - -```bash -cuttle up --cdp-port 9444 --vnc-port 6099 -``` - -- The CLI is **stateless** - pass the *same* ports (and `--name`) to - `status`/`login`/`down` as you gave `up`, or they target the default 9222/6080. -- **Port-shadow gotcha:** `docker run` errors on a docker-vs-docker port clash, - but **not** when a *native* process (e.g. a local CDP shim on 9333) already owns - the host port. `cuttle up` then prints a mapping that is silently dead - your - client hits the other process, not cuttle. Verify with - `lsof -nP -iTCP: -sTCP:LISTEN` (want the owner to be OrbStack/Docker), or - just check `curl http://127.0.0.1:/json/version` reports the engine you - expect. Pick a genuinely free port when unsure. - -## Drive it (drivers + routing) - -cuttle serves standard CDP on `http://127.0.0.1:`. Drive it with a -driver CLI - the briefing lists the ones actually installed, the attach command -for each, and the command that prints that driver's own usage guide. - -- **Attach, never spawn.** Connect to cuttle's running browser and its default - context. Never launch your own Chromium and never create a new profile or - context - logins live in this one session and persist across restarts. cuttle - is a CDP endpoint, not a Chrome binary: never point a driver's - `--executable-path` at it, and do not pass a local `--profile` next to `--cdp` - (drivers reject that - the profile lives in cuttle's container). -- **Prove you are attached before believing a login wall.** A driver that fails - to attach does not error - it quietly drives its *own* fresh browser, and the - symptom is a logged-out page, which looks exactly like a real logged-out state. - `agent-browser connect ` is the known trap: on macOS it can relaunch a - local Chrome (`[agent-browser] relaunched browser`), so pass `--cdp` on every - command instead. To confirm you are on cuttle, check that the driver sees the - session's existing tabs (`playwright-cli attach` prints them) or that - `curl http://127.0.0.1:/json/version` names the same browser. -- **Leave the user's tabs alone.** The session is warm and shared - it may hold a - half-finished login or a page the user is watching in the viewer. Open your - work in a new tab rather than navigating the current one away, and close only - the tabs you opened. -- **One driver at a time.** Every client attached to a cuttle session shares one - browser and one set of tabs; two agents navigating in parallel clobber each - other. Serialize browser work, or give each worker its own identity with - `?fingerprint=` (see [Multi-seed farm](#multi-seed-farm)). -- **Read the site's API, not its DOM.** In a logged-in session the page already - carries the cookies and CSRF token, so an in-page `fetch()` of the site's own - JSON API (via the driver's `eval`) returns clean, complete data. Obfuscated or - lazily-hydrated class names make CSS-selector scraping report "element not - found" even when the content is on screen, and rendered text can silently - differ from what the site actually stored. -- **A logged-in session is the user's real account.** Reads (navigate, snapshot, - extract) are fine. Anything that writes - posting, commenting, reacting, - sending, purchasing, changing settings - needs the user's explicit go-ahead in - the current turn. Draft the content and hand it over; do not submit it. -- **Routing.** The briefing lists installed drivers in priority order; use the - first one (agent-browser by default) unless the user names another - (bu / bu-cli / browseruse = browser-use; playwright-cli). If the named driver - is not installed, use the first listed instead and tell the user you fell back. -- **Driver docs are fetched, not memorized.** Each driver self-documents at a - version-true source; the briefing gives the command per driver. Prefer the - full outputs (with templates/examples) over compact ones, and never rely on - a cached copy of another tool's docs. -- **No driver installed?** Stop and ask the user before installing anything. - Default offer: all three; minimal: just agent-browser. Drivers attach to - cuttle's browser, so skip their own browser downloads. - -Raw CDP libraries work too: - -```javascript -const browser = await chromium.connectOverCDP("http://127.0.0.1:9222"); -const page = browser.contexts()[0].pages()[0]; -``` - -## Log into a site (VNC handoff) - -Log in **once** by hand; the profile keeps you logged in across restarts while a -CDP client drives the same live session. - -```bash -cuttle login https://accounts.google.com -# navigates there, opens the viewer, prints: open the viewer to sign in: http://127.0.0.1:6080/ -``` - -Open the viewer URL, sign in / solve the captcha there, and the CDP connection is -now logged in - VNC and CDP share one browser. This is why cuttle beats a fresh -headless browser for gated sites: the agent hits a wall, hands you the viewer -link, you sign in on the same session, nothing restarts. - -## Lifecycle - -```bash -cuttle status # container + CDP state -cuttle down # graceful stop (SIGTERM -> clean exit); KEEPS the profile -cuttle up # restart the stopped container - logins still there -cuttle down --purge # stop AND remove the container + discard the profile -cuttle up --recreate # destroy any existing container, start fresh -``` - -- **Graceful down matters.** `cuttle down` does `docker stop -t 15` so Chrome - exits clean; that avoids crash-restore junk tabs on next launch. Never - `docker rm -f` a running cuttle - the SIGKILL makes Chrome record a crash. -- **Profile = state.** Logins live in the container filesystem and survive - stop/start. `--purge` / `--recreate` are the only ways to discard them. -- `--keep-profile` (default on) is **fixed at container creation**; passing it (or - `--no-keep-profile`) against an existing container warns and is ignored - use - `--recreate` to change it. -- **`--image` and `--no-vnc` are creation-fixed too.** `--image` against an - existing container warns and is ignored (it will not switch a running container - to another image). `--no-vnc` is ignored *silently*: a container created with it - has no viewer port, yet a later plain `up` still prints a viewer URL that nothing - serves. `cuttle status` shows the container's real image and port bindings; - `--recreate` is the only way to change either, and it discards the profile. -- **Do not reach for `--recreate` on a port error.** If `up` says "container - restarted but CDP on : never came up", suspect a port mismatch first: a - restarted container keeps the ports it was *created* with, and `--recreate` - would discard the logged-in profile. Run `cuttle status` - it prints the real - port bindings and a log tail - then re-run `up` with those ports. - -Also on every subcommand: `--name` (run several side by side), `--no-vnc`, and on -`up` an `--image` override. `cuttle skill` prints this guide to stdout, -always matching the installed CLI. The briefing prints the installed -cuttle-browser version and this guide's frontmatter carries the same number - -if they differ, the copy in your context is stale: rerun `cuttle skill`. - -## Multi-seed farm - -For many isolated identities behind one endpoint - no CLI, no VNC - run the -container directly and select a seed per connection: - -```bash -docker run --rm -p 9222:9222 ghcr.io/glim-sh/cuttle:latest -``` - -``` -http://127.0.0.1:9222?fingerprint=12345 -http://127.0.0.1:9222?fingerprint=12345&timezone=America/New_York&locale=en-US -``` - -Each distinct `fingerprint` seed gets its own isolated Chrome with a stable, -coherent identity; point one CDP client per seed at the seed-parameterized URL. -**Proxy per seed:** pass an authenticated residential proxy on the connect URL - -cuttle strips the inline credentials and answers the proxy `407` over CDP, so -fork binaries that reject inline creds still work. Set proxy, `timezone`, and -`locale` together so the identity is coherent. - -## Engine swap - -Both forks are baked in, selected by `CLOAKBROWSER_BINARY_PATH`: - -```bash -docker run --rm -p 9222:9222 -e CLOAKBROWSER_BINARY_PATH=/opt/clearcote/chrome ghcr.io/glim-sh/cuttle:latest -``` - -- `/opt/clark/chrome` - clark, Chrome 148 (default) -- `/opt/clearcote/chrome` - clearcote, Chrome 149 (fallback) - -## Gotchas - -1. **Headed by default, on purpose.** The image runs `cuttleserve --headless=false` - on a built-in Xvfb; headed Chrome clears escalated anti-bot challenges headless - cannot. Do not force headless. -2. **VNC is loopback-only, no auth.** The viewer serves plain HTTP; the - `-p 127.0.0.1:PORT` mapping is the security boundary. Never bind it publicly. -3. **The engine stealth-presents an older version.** clark's binary is Chromium - 148 but reports `Chrome/146.x` over CDP/UA by design - a coherent, common - fingerprint, not a wrong build. Do not treat the reported version as a defect. -4. **"Logged in" can be false - but so can "logged out".** A CDP context may still - render a login form (geo often defaults to the egress country, which also drives - page *language* on logged-out pages - do not "fix" the language before checking - the signed-in page). The reverse trap is more common: a cookie read that returns - zero cookies is usually the driver probing its *own* blank page, not the site's - tab, while the session is perfectly alive. Verify auth by navigating the tab to - the site and checking what renders - and if the viewer shows you logged in, - trust the viewer. -5. **`cuttle view` is experimental** (CDP-screencast viewer instead of VNC): page - viewport only, no OAuth popups, frame delivery not yet reliable. Use the VNC - viewer (`cuttle up` / `cuttle login`) for real login flows. -6. **Sessions can be IP-bound.** A cookie minted at your real location, replayed - through a proxy in another geo, may force re-login + 2FA. Match the proxy geo - to where the session was created. -7. **"Container running but CDP not answering" / "restarted but CDP never came - up."** Usually a stale container from a *previous* `cuttle up` that failed - because the host port was taken (the failed run leaves a half-created - container with no live port binding). Current `cuttle up` auto-removes such - zombies and starts fresh; if you hit it on an older build, run - `cuttle up --recreate`. `cuttle status` prints a log tail with the real cause. -8. **One failed load is not a verdict on the browser.** Escalated challenges are - dominated by exit-IP reputation, not by the fingerprint: the same seed and - flags can clear in ~7s on a clean exit and take ~200s or fail on a flagged one. - If a page walls you, retry on a *new* identity (a different `?fingerprint=` - seed, and a different proxy exit if you use one) rather than hammering the same - one - do not conclude cuttle is broken from a single attempt. -9. **Chrome's container log noise is not a stealth failure.** `vkCreateInstance: - Found no drivers`, `Automatic fallback to software WebGL`, `Failed to connect to - the bus` (dbus), `Failed to adjust OOM score`, and `GPU stall due to ReadPixels` - are all expected on a headless host and do not mean the browser is broken or - detectable - a probe of the running seed returns a coherent spoofed GPU - (an ANGLE/Direct3D11 vendor-renderer pair), not SwiftShader. Do not "fix" them, - and never add `--enable-unsafe-swiftshader`: it exposes the raw software renderer - and makes the fingerprint worse. -10. **A crash on a `service_worker` target is a client bug, not detection.** Some - Chrome builds report a `service_worker` target without a `browserContextId`, and - older `playwright-core` asserts on it inside a CDP handler, killing the process - mid-run on a page that was loading fine. cuttleserve patches the target shape so - clients do not trip; if you drive cuttle with your own Playwright and still hit - it, pass `serviceWorkers: "block"` to `newContext`. Not a challenge failure. - -## Running on a server - -The amd64 image runs native on any Linux server: - -```bash -docker run -d --restart unless-stopped --name cuttle \ - -p 127.0.0.1:9222:9222 --shm-size=2g ghcr.io/glim-sh/cuttle:latest -``` - -Bind CDP to `127.0.0.1` and reach it over an SSH tunnel -(`ssh -L 9222:localhost:9222 user@server`) - the tunnel is the auth boundary, CDP -has none of its own. `--shm-size=2g` avoids Chrome crashes under load. Add -`-p 127.0.0.1:6080:6080 -e CUTTLE_VNC=1` for the viewer on the server too. - -## See also - -- `README.md` - product overview, quickstart, architecture. -- `AGENTS.md` - repo working rules (vendored code, build, non-negotiables). -- `test/harness.py` (`just smoke`) - neutral CDP smoke: per-seed isolation, - stealth coherence, connection stability. -- `docs/STEALTH-VERIFICATION.md` - confirm a seed presents a coherent identity. diff --git a/SKILL.md b/SKILL.md new file mode 120000 index 0000000..29e3b4a --- /dev/null +++ b/SKILL.md @@ -0,0 +1 @@ +internal/cli/SKILL.md \ No newline at end of file diff --git a/bin/cuttleserve b/bin/cuttleserve deleted file mode 100755 index 2386a0a..0000000 --- a/bin/cuttleserve +++ /dev/null @@ -1,1130 +0,0 @@ -#!/usr/bin/env python3 -"""CDP multiplexer โ€” per-connection fingerprint seeds for stealth Chromium. - -Spawns a separate Chrome process per unique fingerprint seed, routing CDP -connections through a single port. Each seed gets its own browser identity. - -Usage: - cloakserve # default, backward compat - cloakserve --port=9222 # custom port - -Client: - browser = pw.chromium.connect_over_cdp("http://host:9222?fingerprint=12345") - browser = pw.chromium.connect_over_cdp( - "http://host:9222?fingerprint=12345&timezone=America/New_York&locale=en-US" - ) -""" - -from __future__ import annotations - -import asyncio -import ipaddress -import json -import logging -import os -import random -import re -import shutil -import socket -import subprocess -import sys -import time -from dataclasses import dataclass -from urllib.parse import parse_qs, urlparse, urlsplit, urlunsplit - -from pathlib import Path - -import aiohttp -import websockets -from aiohttp import web - -from cloakbrowser.browser import build_args, maybe_resolve_geoip, _resolve_webrtc_args, _normalize_socks_string_url -from cloakbrowser.download import ensure_binary - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - datefmt="%H:%M:%S", -) -logger = logging.getLogger("cloakserve") - -# Args for running Chrome directly (outside Playwright). -# Playwright normally adds its own version of these. -BASE_CHROME_ARGS = [ - "--no-first-run", - "--no-default-browser-check", - "--disable-dev-shm-usage", - "--disable-extensions", - "--disable-popup-blocking", - "--disable-background-networking", - "--metrics-recording-only", - "--ignore-gpu-blocklist", -] - -def _seed_profile_defaults(user_data_dir: str) -> None: - """DuckDuckGo as default search on first launch (same seeding CBM does). - Only written while the profile is brand new - Chrome owns the file after. - Tab/session restore is handled correctly by shutting Chrome down cleanly - (SIGTERM in _cleanup_process), NOT by forging clean-exit flags here.""" - default_dir = os.path.join(user_data_dir, "Default") - prefs_path = os.path.join(default_dir, "Preferences") - if os.path.exists(prefs_path): - return - os.makedirs(default_dir, exist_ok=True) - prefs = { - "default_search_provider_data": { - "template_url_data": { - "keyword": "duckduckgo.com", - "short_name": "DuckDuckGo", - "url": "https://duckduckgo.com/?q={searchTerms}", - "suggestions_url": "https://duckduckgo.com/ac/?q={searchTerms}&type=list", - "favicon_url": "https://duckduckgo.com/favicon.ico", - } - }, - "default_search_provider": {"enabled": True}, - } - with open(prefs_path, "w", encoding="utf-8") as f: - json.dump(prefs, f) - - -BASE_CDP_PORT = 5100 - -SAFE_SEED_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") -RESERVED_SEEDS = {"__default__"} -TRUSTED_WS_ORIGINS = {"devtools://devtools", "chrome-devtools://devtools"} - - -def _host_port_from_netloc(netloc: str, default_port: int) -> tuple[str, int] | None: - """Return a normalized (host, port) pair for an Origin/Host netloc.""" - if "," in netloc: - return None - try: - parsed = urlparse(f"//{netloc.strip()}") - authority = parsed.netloc.rsplit("@", 1)[-1] - if ( - not parsed.hostname - or parsed.username is not None - or parsed.password is not None - or authority.endswith(":") - or parsed.path - or parsed.params - or parsed.query - or parsed.fragment - ): - return None - return (parsed.hostname.lower(), parsed.port if parsed.port is not None else default_port) - except ValueError: - return None - - -def _is_loopback_host(hostname: str) -> bool: - """Return True for localhost and loopback IP literals.""" - hostname = hostname.strip("[]").rstrip(".").lower() - if hostname == "localhost": - return True - try: - return ipaddress.ip_address(hostname).is_loopback - except ValueError: - return False - - -def _origin_is_allowed( - origin: str | None, - host: str | None, - request_scheme: str = "http", -) -> bool: - """Return True when a WebSocket Origin is safe to proxy to local CDP.""" - if origin is None: - # Playwright/Puppeteer and other non-browser CDP clients commonly omit - # Origin. Keep those clients working while rejecting browser-origin CSRF. - return True - - origin = origin.strip() - if not origin or origin.lower() == "null": - return False - if origin in TRUSTED_WS_ORIGINS: - return True - - try: - parsed = urlparse(origin) - except ValueError: - return False - - if parsed.scheme not in ("http", "https"): - return False - if parsed.path or parsed.params or parsed.query or parsed.fragment: - return False - - origin_default_port = 443 if parsed.scheme == "https" else 80 - request_scheme = request_scheme.split(",", 1)[0].strip().lower() - request_default_port = 443 if request_scheme in ("https", "wss") else 80 - origin_host = _host_port_from_netloc(parsed.netloc, origin_default_port) - request_host = _host_port_from_netloc(host or "", request_default_port) - if origin_host is None or request_host is None: - return False - if not _is_loopback_host(request_host[0]): - return False - return origin_host == request_host - - -def _reject_untrusted_origin(request: web.Request) -> web.Response | None: - """Reject browser-origin WebSocket upgrades that would expose local CDP.""" - origin = request.headers.get("Origin") - host = request.headers.get("Host") - scheme = request.headers.get("X-Forwarded-Proto", getattr(request, "scheme", "http")) - if _origin_is_allowed(origin, host, request_scheme=scheme): - return None - logger.warning("Rejected CDP WebSocket from untrusted Origin %r for Host %r", origin, host) - return web.Response(status=403, text="Forbidden: untrusted WebSocket origin\n") - - -# --------------------------------------------------------------------------- -# ChromeProcess โ€” one running Chrome instance -# --------------------------------------------------------------------------- - -def _split_proxy_auth(proxy: str) -> tuple[str, str | None, str | None]: - """Split inline credentials out of an http(s) proxy URL. - - Stock Chromium (and the free stealth-Chromium forks) does NOT accept - credentials in `--proxy-server=http://user:pass@host`; only CloakBrowser's - proprietary binary has the inline-credential patch. So for a credentialed - http(s) proxy, return the cred-stripped URL (for `--proxy-server`) plus the - username/password (answered over CDP `Fetch.continueWithAuth`). socks and - cred-less proxies pass through unchanged (None creds).""" - try: - parts = urlsplit(proxy) - except Exception: - return proxy, None, None - if parts.scheme not in ("http", "https") or parts.username is None: - return proxy, None, None - host = parts.hostname or "" - if parts.port: - host = f"{host}:{parts.port}" - stripped = urlunsplit((parts.scheme, host, parts.path, parts.query, parts.fragment)) - return stripped, parts.username, parts.password or "" - - -@dataclass -class ChromeProcess: - seed: str - process: subprocess.Popen - cdp_port: int - user_data_dir: str - timezone: str | None = None - locale: str | None = None - proxy: str | None = None - - -# --------------------------------------------------------------------------- -# ChromePool โ€” manages multiple Chrome processes keyed by seed -# --------------------------------------------------------------------------- - -class ChromePool: - def __init__( - self, - binary: str, - global_args: list[str], - headless: bool, - data_dir: str = "/tmp/cloakserve", - default_seed: str | None = None, - default_locale: str | None = None, - default_timezone: str | None = None, - idle_timeout: float = 0.0, - keep_profile: bool = False, - ): - self._binary = binary - self._global_args = global_args - self._headless = headless - self._data_dir = data_dir - self._default_seed = default_seed - self._default_locale = default_locale - self._default_timezone = default_timezone - self._idle_timeout = idle_timeout - self._keep_profile = keep_profile - self._processes: dict[str, ChromeProcess] = {} - self._default: ChromeProcess | None = None - self._locks: dict[str, asyncio.Lock] = {} - self._next_port = BASE_CDP_PORT - # Connection refcounting for status reporting - self._connections: dict[str, int] = {} - self._idle_tasks: dict[str, asyncio.Task] = {} - - def _get_lock(self, seed: str) -> asyncio.Lock: - if seed not in self._locks: - self._locks[seed] = asyncio.Lock() - return self._locks[seed] - - def _safe_rmtree(self, path: str) -> None: - if self._keep_profile: - logger.info("keep-profile: preserving %s", path) - return - resolved = Path(path).resolve() - data_resolved = Path(self._data_dir).resolve() - if resolved == data_resolved or not resolved.is_relative_to(data_resolved): - logger.error("Refusing to delete path outside data_dir: %s", resolved) - return - shutil.rmtree(path, True) - - def _allocate_port(self) -> int: - """Find a free port starting from _next_port.""" - for _ in range(100): - port = self._next_port - self._next_port += 1 - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", port)) - return port - except OSError: - continue - raise RuntimeError("No free ports available for Chrome CDP") - - def connect(self, seed_key: str) -> None: - """Increment connection refcount for a seed.""" - self._cancel_idle_cleanup(seed_key) - self._connections[seed_key] = self._connections.get(seed_key, 0) + 1 - - def disconnect(self, seed_key: str) -> None: - """Decrement connection refcount for a seed.""" - count = self._connections.get(seed_key, 0) - 1 - if count <= 0: - self._connections.pop(seed_key, None) - self._schedule_idle_cleanup(seed_key) - else: - self._connections[seed_key] = count - - def _cancel_idle_cleanup(self, seed_key: str) -> None: - task = self._idle_tasks.pop(seed_key, None) - if task is None or task.done(): - return - try: - current_task = asyncio.current_task() - except RuntimeError: - current_task = None - if task is not current_task: - task.cancel() - - def _discard_idle_task(self, seed_key: str, task: asyncio.Task) -> None: - if self._idle_tasks.get(seed_key) is task: - self._idle_tasks.pop(seed_key, None) - - def _schedule_idle_cleanup(self, seed_key: str) -> None: - if self._idle_timeout <= 0 or seed_key not in self._processes: - return - - self._cancel_idle_cleanup(seed_key) - try: - loop = asyncio.get_running_loop() - except RuntimeError: - return - - task = loop.create_task( - self._cleanup_after_idle(seed_key, self._idle_timeout), - name=f"cloakserve-idle-cleanup-{seed_key}", - ) - self._idle_tasks[seed_key] = task - task.add_done_callback(lambda done_task: self._discard_idle_task(seed_key, done_task)) - - async def _cleanup_after_idle(self, seed_key: str, timeout: float) -> None: - try: - await asyncio.sleep(timeout) - if self._connections.get(seed_key, 0) > 0 or seed_key not in self._processes: - return - logger.info("Cleaning up idle Chrome process (seed=%s)", seed_key) - await self._cleanup_process(seed_key) - except asyncio.CancelledError: - raise - except Exception: - logger.exception("Idle cleanup failed for seed=%s", seed_key) - - async def get_or_launch( - self, - seed: str | None, - extra_args: list[str] | None = None, - timezone: str | None = None, - locale: str | None = None, - proxy: str | None = None, - geoip: bool = False, - ) -> ChromeProcess: - """Get existing or launch new Chrome process for a seed.""" - # Apply CLI defaults when query params don't provide values - if seed is None and self._default_seed: - seed = self._default_seed - if locale is None: - locale = self._default_locale - if timezone is None: - timezone = self._default_timezone - - # No seed = default shared process - if seed is None: - seed_key = "__default__" - actual_seed = str(random.randint(10000, 99999)) - else: - if not SAFE_SEED_RE.match(seed) or seed in RESERVED_SEEDS: - raise web.HTTPBadRequest( - text=json.dumps({"error": "Invalid fingerprint seed"}), - content_type="application/json", - ) - seed_key = seed - actual_seed = seed - - lock = self._get_lock(seed_key) - async with lock: - # Check if already running (including default fast-path) - if seed_key in self._processes: - proc = self._processes[seed_key] - if proc.process.poll() is None: - if seed_key in self._idle_tasks: - self._schedule_idle_cleanup(seed_key) - if any([extra_args, timezone, locale, proxy, geoip]): - logger.warning( - "Seed %s already running (port %d, tz=%s, locale=%s, proxy=%s) โ€” " - "ignoring new params (first-launch wins)", - seed_key, proc.cdp_port, - proc.timezone, proc.locale, proc.proxy, - ) - return proc - # Dead โ€” clean up - await self._cleanup_process(seed_key) - - # Resolve geoip if requested - exit_ip = None - if geoip and proxy: - timezone, locale, exit_ip = maybe_resolve_geoip(True, proxy, timezone, locale) - - # Build Chrome args via shared logic - fp_extra = [f"--fingerprint={actual_seed}"] - if extra_args: - fp_extra.extend(extra_args) - if proxy: - # Strip inline creds: the fork binaries reject them on - # --proxy-server (ERR_NO_SUPPORTED_PROXIES). We answer the proxy - # 407 over CDP at the multiplexer instead (see proxy_cdp_websocket - # / _inject_proxy_auth). geoip above still uses the credentialed - # `proxy` to resolve the exit IP; the full proxy (with creds) is - # stored on the ChromeProcess so the ws proxy can recover them. - stripped, _, _ = _split_proxy_auth(proxy) - fp_extra.append(f"--proxy-server={_normalize_socks_string_url(stripped)}") - - # WebRTC IP spoofing: resolve auto, inject geoip exit IP - fp_extra = _resolve_webrtc_args(fp_extra, proxy) - if exit_ip and not any(a.startswith("--fingerprint-webrtc-ip") for a in (fp_extra or [])): - fp_extra = list(fp_extra or []) - fp_extra.append(f"--fingerprint-webrtc-ip={exit_ip}") - - # Fork-binary parity (clark/clearcote via CLOAKBROWSER_BINARY_PATH). - # cloakbrowser's build_args is tuned for the Pro binary and omits flags - # the fork binaries REQUIRE - replicate clark's own launcher set - # (clarkbrowser.config.get_default_stealth_args). Most critical: an - # explicit --user-agent so the HTTP UA header matches the JS - # navigator.userAgent (mismatch is a textbook bot tell); the ungoogled - # canvas/client-rects noise switches clark enables by default (patch - # #50 forwards them to renderers); UA-CH brand/platform-version - # coherence; a Windows font dir (real Arial/Calibri/Segoe UI names, - # see /opt/winfonts) so the Windows profile is coherent on Linux; the - # Accept-Language header; and a residential Network Information profile. - if os.environ.get("CLOAKBROWSER_BINARY_PATH"): - lang = locale or "en-US" - base = lang.split("-", 1)[0] - fp_extra += [ - "--fingerprint-platform=windows", - "--fingerprint-platform-version=19.0.0", - "--fingerprint-brand=Chrome", - "--fingerprint-brand-version=148.0.0.0", - "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", - "--fingerprint-fonts-dir=/opt/winfonts", - "--fingerprinting-client-rects-noise", - "--fingerprinting-canvas-measuretext-noise", - "--fingerprinting-canvas-image-data-noise", - f"--accept-lang={lang},{base}" if base != lang else f"--accept-lang={lang}", - ] - if proxy: - fp_extra.append("--fingerprint-network-profile=residential") - - chrome_args = build_args( - stealth_args=True, - extra_args=fp_extra, - timezone=timezone, - locale=locale, - headless=self._headless, - ) - - # Allocate port and user data dir - port = self._allocate_port() - user_data_dir = os.path.join(self._data_dir, seed_key) - os.makedirs(user_data_dir, exist_ok=True) - _seed_profile_defaults(user_data_dir) - - full_args = ( - [self._binary] - + BASE_CHROME_ARGS - + chrome_args - + self._global_args - + [ - f"--remote-debugging-port={port}", - "--remote-debugging-address=127.0.0.1", - f"--user-data-dir={user_data_dir}", - ] - ) - - logger.info("Launching Chrome (seed=%s, port=%d)", actual_seed, port) - process = subprocess.Popen( - full_args, - stdout=subprocess.DEVNULL, - ) - - # Wait for CDP to be ready - if not await self._wait_for_cdp(port): - process.kill() - await asyncio.to_thread(process.wait, timeout=5) - await asyncio.to_thread(self._safe_rmtree, user_data_dir) - raise web.HTTPBadGateway( - text=json.dumps({"error": "Chrome failed to start"}), - content_type="application/json", - ) - - cp = ChromeProcess( - seed=actual_seed, - process=process, - cdp_port=port, - user_data_dir=user_data_dir, - timezone=timezone, - locale=locale, - proxy=proxy, - ) - self._processes[seed_key] = cp - - if seed is None: - self._default = cp - - logger.info("Chrome ready (seed=%s, port=%d, pid=%d)", actual_seed, port, process.pid) - return cp - - async def _cleanup_process(self, key: str) -> None: - """Terminate a Chrome process and clean up.""" - self._cancel_idle_cleanup(key) - proc = self._processes.pop(key, None) - if not proc: - return - if proc.process.poll() is None: - proc.process.terminate() - try: - await asyncio.to_thread(proc.process.wait, timeout=5) - except subprocess.TimeoutExpired: - proc.process.kill() - await asyncio.to_thread(self._safe_rmtree, proc.user_data_dir) - if self._default is proc: - self._default = None - self._locks.pop(key, None) - self._connections.pop(key, None) - - async def shutdown(self) -> None: - """Terminate all Chrome processes.""" - idle_tasks = list(self._idle_tasks.values()) - self._idle_tasks.clear() - for task in idle_tasks: - if not task.done(): - task.cancel() - if idle_tasks: - await asyncio.gather(*idle_tasks, return_exceptions=True) - for key in list(self._processes.keys()): - await self._cleanup_process(key) - logger.info("All Chrome processes terminated") - - @staticmethod - async def _wait_for_cdp(port: int, timeout: float = 10.0) -> bool: - """Poll Chrome's /json/version until ready.""" - deadline = time.monotonic() + timeout - delay = 0.1 - session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=1) - ) - try: - while time.monotonic() < deadline: - try: - async with session.get( - f"http://127.0.0.1:{port}/json/version" - ) as resp: - if resp.status == 200: - return True - except Exception: - pass - await asyncio.sleep(delay) - delay = min(delay * 2, 1.0) - return False - finally: - await session.close() - - -# --------------------------------------------------------------------------- -# Query param parsing -# --------------------------------------------------------------------------- - -# Params that need special handling (not simple --fingerprint-{name}= mapping) -SPECIAL_PARAMS = {"fingerprint", "proxy", "geoip", "locale", "timezone"} - - -def parse_connection_params(query_string: str) -> dict: - """Parse query params into connection config.""" - qs = parse_qs(query_string, keep_blank_values=False) - - result: dict = { - "seed": None, - "timezone": None, - "locale": None, - "proxy": None, - "geoip": False, - "extra_args": [], - } - - for key, values in qs.items(): - val = values[0] - if key == "fingerprint": - result["seed"] = val - elif key == "timezone": - result["timezone"] = val - elif key == "locale": - result["locale"] = val - elif key == "proxy": - result["proxy"] = val - elif key == "geoip": - result["geoip"] = val.lower() in ("true", "1", "yes") - elif key not in SPECIAL_PARAMS: - # Generic fingerprint param: map to --fingerprint-{key}={val} - result["extra_args"].append(f"--fingerprint-{key}={val}") - - return result - - -# --------------------------------------------------------------------------- -# HTTP handlers -# --------------------------------------------------------------------------- - -def _ws_scheme(request: web.Request) -> str: - """Return 'wss' if client connected via HTTPS (e.g. TLS-terminating proxy), else 'ws'.""" - proto = request.headers.get("X-Forwarded-Proto", request.scheme) - proto = proto.split(",", 1)[0].strip().lower() - return "wss" if proto == "https" else "ws" - - -def _external_host(request: web.Request) -> str: - """Return the public host to use in rewritten CDP WebSocket URLs.""" - fallback_host = request.headers.get("Host") or f"localhost:{request.app['port']}" - forwarded_host = request.headers.get("X-Forwarded-Host") - if forwarded_host: - public_host = forwarded_host.split(",", 1)[0].strip() - if public_host: - return public_host - return fallback_host - - -async def handle_root(request: web.Request) -> web.Response: - """Health check / process status.""" - pool: ChromePool = request.app["pool"] - processes = {} - for key, proc in pool._processes.items(): - if proc.process.poll() is None: - processes[key] = { - "pid": proc.process.pid, - "port": proc.cdp_port, - "seed": proc.seed, - "connections": pool._connections.get(key, 0), - "idle_cleanup_pending": key in pool._idle_tasks, - "timezone": proc.timezone, - "locale": proc.locale, - "proxy": proc.proxy, - } - return web.json_response({ - "status": "ok", - "active": len(processes), - "idle_timeout": pool._idle_timeout, - "processes": processes, - }) - - -async def handle_json_version(request: web.Request) -> web.Response: - """Proxy /json/version with optional per-seed routing.""" - pool: ChromePool = request.app["pool"] - params = parse_connection_params(request.query_string) - - cp = await pool.get_or_launch( - seed=params["seed"], - extra_args=params["extra_args"] or None, - timezone=params["timezone"], - locale=params["locale"], - proxy=params["proxy"], - geoip=params["geoip"], - ) - - try: - async with aiohttp.ClientSession() as session: - async with session.get( - f"http://127.0.0.1:{cp.cdp_port}/json/version", - timeout=aiohttp.ClientTimeout(total=5), - ) as resp: - data = await resp.json() - except Exception as exc: - logger.error("Failed to reach Chrome CDP (port %d): %s", cp.cdp_port, exc) - return web.json_response({"error": "CDP endpoint unreachable"}, status=502) - - # Rewrite webSocketDebuggerUrl to route through our multiplexer - host = _external_host(request) - seed_key = params["seed"] - if seed_key: - ws_path = f"fingerprint/{seed_key}/devtools/browser" - else: - ws_path = "devtools/browser" - - # Extract the browser GUID from Chrome's original URL - orig_ws = data.get("webSocketDebuggerUrl", "") - guid = orig_ws.rsplit("/", 1)[-1] if "/devtools/" in orig_ws else "" - - scheme = _ws_scheme(request) - data["webSocketDebuggerUrl"] = f"{scheme}://{host}/{ws_path}/{guid}" - return web.json_response(data) - - -async def handle_json_list(request: web.Request) -> web.Response: - """Proxy /json/list with per-seed routing. Rewrites all entries.""" - pool: ChromePool = request.app["pool"] - params = parse_connection_params(request.query_string) - - cp = await pool.get_or_launch( - seed=params["seed"], - extra_args=params["extra_args"] or None, - timezone=params["timezone"], - locale=params["locale"], - proxy=params["proxy"], - geoip=params["geoip"], - ) - - try: - async with aiohttp.ClientSession() as session: - async with session.get( - f"http://127.0.0.1:{cp.cdp_port}/json/list", - timeout=aiohttp.ClientTimeout(total=5), - ) as resp: - data = await resp.json() - except Exception as exc: - logger.error("Failed to reach Chrome CDP (port %d): %s", cp.cdp_port, exc) - return web.json_response({"error": "CDP endpoint unreachable"}, status=502) - - host = _external_host(request) - scheme = _ws_scheme(request) - seed_key = params["seed"] - - for entry in data: - if "webSocketDebuggerUrl" in entry: - ws_tail = entry["webSocketDebuggerUrl"].split("/devtools/")[-1] - if seed_key: - entry["webSocketDebuggerUrl"] = ( - f"{scheme}://{host}/fingerprint/{seed_key}/devtools/{ws_tail}" - ) - else: - entry["webSocketDebuggerUrl"] = f"{scheme}://{host}/devtools/{ws_tail}" - - return web.json_response(data) - - -# --------------------------------------------------------------------------- -# WebSocket proxy -# --------------------------------------------------------------------------- - -# Injected CDP command ids for transparent proxy-auth start here - far above any -# id a client would realistically use, so their responses are recognizable and -# swallowed rather than forwarded to the client. -_INJECTED_ID_BASE = 2_000_000_000 - -# Synthetic browserContextId stamped onto default-context service_worker targets -# (see _stamp_sw_context). Any truthy value works - playwright looks it up, misses, -# and falls back to its default context; it never needs to resolve to a real id. -_SYNTH_BROWSER_CONTEXT_ID = "0000000000000000000000000000CA5E" - - -def _stamp_sw_context(data: str) -> str: - """Chrome 148 reports a site's service_worker target under the default browser - context with an EMPTY browserContextId. playwright-core's connectOverCDP asserts - that field is truthy in its Target.attachedToTarget handler, and the throw is - uncaught - it kills the client process (repro: any page that registers a service - worker). Stamp a synthetic - id so the assert passes; playwright then falls back to its default context and - handles the SW normally. The browser and page stay fully authentic - the service - worker actually registers and runs, and nothing in navigator is patched. Only - service_worker attachedToTarget frames with a missing id are touched.""" - if '"Target.attachedToTarget"' not in data or '"service_worker"' not in data: - return data - try: - msg = json.loads(data) - except Exception: - return data - if msg.get("method") != "Target.attachedToTarget": - return data - target_info = msg.get("params", {}).get("targetInfo", {}) - if target_info.get("type") != "service_worker" or target_info.get("browserContextId"): - return data - target_info["browserContextId"] = _SYNTH_BROWSER_CONTEXT_ID - return json.dumps(msg) - - -def _rewrite_fetch_enable(data: str) -> str: - """Add handleAuthRequests to a client's Fetch.enable so Chrome surfaces proxy - 407s as Fetch.authRequired on the client's own session. Other frames pass - through untouched.""" - if '"Fetch.enable"' not in data: - return data - try: - msg = json.loads(data) - except Exception: - return data - if msg.get("method") != "Fetch.enable": - return data - params = msg.setdefault("params", {}) - if params.get("handleAuthRequests") is True: - return data - params["handleAuthRequests"] = True - return json.dumps(msg) - - -def _handle_proxy_auth( - data: str, - injected_ids: set[int], - cmd_id: int, - username: str, - password: str, -) -> tuple[bool, str | None]: - """Inspect a Chrome->client frame. Returns (swallow, command_to_send): - - a response to one of our injected commands -> (True, None), swallow it; - - a Fetch.authRequired -> (True, continueWithAuth json), answer it and swallow - it (the client never asked for auth handling); - - anything else -> (False, None), forward normally.""" - try: - msg = json.loads(data) - except Exception: - return False, None - mid = msg.get("id") - if mid is not None and mid in injected_ids: - injected_ids.discard(mid) - if "error" in msg: - logger.warning("Proxy-auth: continueWithAuth failed: %s", msg["error"]) - return True, None - if msg.get("method") != "Fetch.authRequired": - return False, None - params = msg.get("params", {}) - is_proxy = params.get("authChallenge", {}).get("source") == "Proxy" - response = ( - {"response": "ProvideCredentials", "username": username, "password": password} - if is_proxy - else {"response": "Default"} - ) - cmd: dict = { - "id": cmd_id, - "method": "Fetch.continueWithAuth", - "params": {"requestId": params.get("requestId"), "authChallengeResponse": response}, - } - if msg.get("sessionId"): - cmd["sessionId"] = msg["sessionId"] - injected_ids.add(cmd_id) - return True, json.dumps(cmd) - - -async def proxy_cdp_websocket( - client_ws: web.WebSocketResponse, - target_url: str, - label: str, - proxy_user: str | None = None, - proxy_pass: str | None = None, -) -> None: - """Bidirectional WebSocket proxy between client and Chrome CDP. - - When the seed's Chrome runs behind a credential-stripped --proxy-server (fork - binaries reject inline creds), transparently answer proxy 407s: the client's - own `Fetch.enable` is rewritten to also handleAuthRequests, and the resulting - `Fetch.authRequired` events are intercepted here and answered with the stored - credentials over CDP โ€” never surfaced to the client. This rides the client's - OWN Fetch session, so it works for HTTPS CONNECT and doesn't conflict with the - client's request interception (page.route).""" - inject = proxy_user is not None - try: - async with websockets.connect( - target_url, max_size=None, ping_interval=None, ping_timeout=None, - ) as cdp_ws: - logger.info("%s: connected to %s", label, target_url) - send_lock = asyncio.Lock() - injected_ids: set[int] = set() - next_injected = _INJECTED_ID_BASE - - async def cdp_send(data: str) -> None: - async with send_lock: - await cdp_ws.send(data) - - async def client_to_cdp(): - try: - async for msg in client_ws: - if msg.type == aiohttp.WSMsgType.TEXT: - data = msg.data - if inject: - data = _rewrite_fetch_enable(data) - await cdp_send(data) - elif msg.type == aiohttp.WSMsgType.BINARY: - await cdp_send(msg.data) - elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSING, aiohttp.WSMsgType.CLOSED): - break - except Exception as exc: - logger.debug("%s [c->cdp]: %s", label, exc) - - async def cdp_to_client(): - nonlocal next_injected - try: - async for msg in cdp_ws: - if inject and isinstance(msg, str): - handled, cmd = _handle_proxy_auth( - msg, injected_ids, next_injected, proxy_user, proxy_pass, - ) - if cmd is not None: - next_injected += 1 - await cdp_send(cmd) - if handled: - continue # swallow authRequired / our own responses - if isinstance(msg, str): - await client_ws.send_str(_stamp_sw_context(msg)) - else: - await client_ws.send_bytes(msg) - except Exception as exc: - logger.debug("%s [cdp->c]: %s", label, exc) - - c2d = asyncio.create_task(client_to_cdp(), name="c2d") - d2c = asyncio.create_task(cdp_to_client(), name="d2c") - done, pending = await asyncio.wait( - [c2d, d2c], return_when=asyncio.FIRST_COMPLETED, - ) - for task in pending: - task.cancel() - logger.info("%s: disconnected", label) - - except Exception as exc: - logger.error("%s error: %s", label, exc) - - -async def handle_ws_default(request: web.Request) -> web.StreamResponse: - """WebSocket proxy for default (no-seed) Chrome: /devtools/{type}/{guid}""" - rejected = _reject_untrusted_origin(request) - if rejected is not None: - return rejected - - pool: ChromePool = request.app["pool"] - path = request.match_info.get("path", "") - - cp = await pool.get_or_launch(seed=None) - - ws = web.WebSocketResponse() - await ws.prepare(request) - - _, p_user, p_pass = _split_proxy_auth(cp.proxy) if cp.proxy else (None, None, None) - pool.connect("__default__") - try: - target_url = f"ws://127.0.0.1:{cp.cdp_port}/devtools/{path}" - await proxy_cdp_websocket(ws, target_url, f"CDP default [{path}]", p_user, p_pass) - finally: - pool.disconnect("__default__") - return ws - - -async def handle_ws_seed(request: web.Request) -> web.StreamResponse: - """WebSocket proxy for seed-specific Chrome: /fingerprint/{seed}/devtools/{type}/{guid}""" - rejected = _reject_untrusted_origin(request) - if rejected is not None: - return rejected - - pool: ChromePool = request.app["pool"] - seed = request.match_info["seed"] - path = request.match_info.get("path", "") - - cp = await pool.get_or_launch(seed=seed) - - ws = web.WebSocketResponse() - await ws.prepare(request) - - _, p_user, p_pass = _split_proxy_auth(cp.proxy) if cp.proxy else (None, None, None) - pool.connect(seed) - try: - target_url = f"ws://127.0.0.1:{cp.cdp_port}/devtools/{path}" - await proxy_cdp_websocket(ws, target_url, f"CDP seed={seed} [{path}]", p_user, p_pass) - finally: - pool.disconnect(seed) - return ws - - -async def on_shutdown(app: web.Application) -> None: - await app["pool"].shutdown() - - -# --------------------------------------------------------------------------- -# CLI arg parsing -# --------------------------------------------------------------------------- - -def _in_container() -> bool: - """True when running inside a container (docker, podman, or k8s/containerd). - - Governs the bind host and data dir. The plain-file markers /.dockerenv and - /run/.containerenv are docker- and podman-only, and BOTH are absent under - Kubernetes+containerd - which silently pinned the CDP listener to loopback and - refused every cross-pod client (Docker Desktop's port proxy hid it in local - dev). Also detect k8s via KUBERNETES_SERVICE_HOST and a container cgroup. - """ - if os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv"): - return True - if os.environ.get("KUBERNETES_SERVICE_HOST"): - return True - try: - with open("/proc/1/cgroup", encoding="utf-8") as f: - cgroup = f.read() - except OSError: - return False - return any(m in cgroup for m in ("kubepods", "docker", "containerd", "crio")) - - -def _default_data_dir() -> str: - """Smart default: container โ†’ /tmp/cloakserve, bare metal โ†’ ~/.cloakbrowser/cloakserve.""" - if _in_container(): - return "/tmp/cloakserve" - return str(Path.home() / ".cloakbrowser" / "cloakserve") - - -def _parse_idle_timeout(value: str) -> float: - value = value.strip() - if value.lower() in {"0", "false", "off", "none", "disabled"}: - return 0.0 - timeout = float(value) - if timeout < 0: - raise ValueError("--idle-timeout must be greater than or equal to 0") - return timeout - - -def _default_idle_timeout() -> float: - value = os.environ.get("CLOAKSERVE_IDLE_TIMEOUT") - if value is None: - return 0.0 - return _parse_idle_timeout(value) - - -def parse_cli_args(argv: list[str]) -> tuple[dict, list[str]]: - """Parse cloakserve-specific args, return (config, passthrough_args). - - --fingerprint, --fingerprint-locale, and --fingerprint-timezone are - extracted into config defaults so they route through build_args() - (e.g. locale needs both --lang and --fingerprint-locale). - Query-string params override these defaults per-connection. - """ - config: dict = { - "port": 9222, - "headless": True, - "data_dir": None, - "default_seed": None, - "default_locale": None, - "default_timezone": None, - "idle_timeout": _default_idle_timeout(), - "keep_profile": False, - } - passthrough = [] - # Flags consumed by cloakserve (not passed to Chrome) - consumed_prefixes = ( - "--port=", - "--data-dir=", - "--idle-timeout=", - "--remote-debugging-port=", - "--remote-debugging-address=", - ) - - for arg in argv: - if arg.startswith("--port="): - config["port"] = int(arg.split("=", 1)[1]) - elif arg.startswith("--data-dir="): - config["data_dir"] = arg.split("=", 1)[1] - elif arg.startswith("--idle-timeout="): - config["idle_timeout"] = _parse_idle_timeout(arg.split("=", 1)[1]) - elif arg == "--headless=false" or arg == "--headless=False": - config["headless"] = False - passthrough.append(arg) - elif arg == "--keep-profile": - config["keep_profile"] = True - elif arg.startswith(consumed_prefixes): - pass # Strip these silently - # Route through build_args() so companion flags are set correctly - elif arg.startswith("--fingerprint-locale="): - config["default_locale"] = arg.split("=", 1)[1] - elif arg.startswith("--fingerprint-timezone="): - config["default_timezone"] = arg.split("=", 1)[1] - elif arg.startswith("--fingerprint="): - config["default_seed"] = arg.split("=", 1)[1] - else: - passthrough.append(arg) - - if config["data_dir"] is None: - config["data_dir"] = _default_data_dir() - - return config, passthrough - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - binary = ensure_binary() - config, global_args = parse_cli_args(sys.argv[1:]) - - if config["default_seed"] and ( - not SAFE_SEED_RE.match(config["default_seed"]) - or config["default_seed"] in RESERVED_SEEDS - ): - logger.error("Invalid --fingerprint seed: %s", config["default_seed"]) - sys.exit(1) - - pool = ChromePool( - binary=binary, - global_args=global_args, - headless=config["headless"], - data_dir=config["data_dir"], - default_seed=config["default_seed"], - default_locale=config["default_locale"], - default_timezone=config["default_timezone"], - idle_timeout=config["idle_timeout"], - keep_profile=config["keep_profile"], - ) - - app = web.Application() - app["pool"] = pool - app["port"] = config["port"] - - # Routes - app.router.add_get("/", handle_root) - app.router.add_get("/json/version", handle_json_version) - app.router.add_get("/json/version/", handle_json_version) - app.router.add_get("/json/list", handle_json_list) - app.router.add_get("/json/list/", handle_json_list) - app.router.add_get("/json", handle_json_list) - app.router.add_get("/json/", handle_json_list) - - # WebSocket routes โ€” seed-specific (must be before default to match first) - app.router.add_get("/fingerprint/{seed}/devtools/{path:.+}", handle_ws_seed) - # WebSocket routes โ€” default (no seed) - app.router.add_get("/devtools/{path:.+}", handle_ws_default) - - app.on_shutdown.append(on_shutdown) - - port = config["port"] - logger.info("CloakBrowser CDP multiplexer starting on port %d", port) - logger.info( - "Connect: playwright.chromium.connect_over_cdp(" - "\"http://localhost:%d?fingerprint=\")", - port, - ) - - # Bind 0.0.0.0 in a container so cross-pod/host clients can reach the CDP - # multiplexer; stay loopback-only on bare metal. CUTTLESERVE_HOST overrides. - host = os.environ.get("CUTTLESERVE_HOST") or ("0.0.0.0" if _in_container() else "127.0.0.1") - web.run_app(app, host=host, port=port, print=None) - - -if __name__ == "__main__": - main() diff --git a/cmd/cuttle/main.go b/cmd/cuttle/main.go new file mode 100644 index 0000000..e8be9ed --- /dev/null +++ b/cmd/cuttle/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "fmt" + "os" + + "github.com/glim-sh/cuttle/internal/cli" + _ "github.com/glim-sh/cuttle/internal/serve" +) + +func main() { + if err := cli.Execute(); err != nil { + fmt.Fprintln(os.Stderr, "cuttle:", err) + os.Exit(1) + } +} diff --git a/cuttle/SKILL.md b/cuttle/SKILL.md deleted file mode 120000 index d5d81a4..0000000 --- a/cuttle/SKILL.md +++ /dev/null @@ -1 +0,0 @@ -../SKILL.md \ No newline at end of file diff --git a/cuttle/__init__.py b/cuttle/__init__.py deleted file mode 100644 index f2f24db..0000000 --- a/cuttle/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""cuttle - authored host-side tooling. - -This package is our own code (linted + typed), separate from the vendored -upstream subset under `vendor/cloakbrowser/`. It drives a running cuttle -container from the outside over Docker + CDP: - -- `cuttle.cli` - the `cuttle` command (up / login / view / status / down) -- `cuttle.view` - experimental CDP-screencast viewer (alternative to VNC) - -Nothing here is imported by `bin/cuttleserve` or baked into the runtime path; -the container needs only `cloakbrowser`. -""" diff --git a/cuttle/cdp.py b/cuttle/cdp.py deleted file mode 100644 index 6da7f6c..0000000 --- a/cuttle/cdp.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Minimal CDP client helpers shared by the CLI and the screencast viewer. - -Just enough Chrome DevTools Protocol to list targets, pick the page a human -would want to see, and issue commands - no playwright, no heavy client. Talks -to cuttleserve's HTTP `/json` list and the per-page WebSocket it hands back. -""" - -from __future__ import annotations - -import json -import urllib.request -from typing import Any - -import websockets - - -def list_targets(cdp_port: int, timeout: float = 5.0) -> list[dict[str, Any]]: - with urllib.request.urlopen(f"http://127.0.0.1:{cdp_port}/json", timeout=timeout) as resp: - data = json.loads(resp.read()) - return data if isinstance(data, list) else [] - - -def pick_page(targets: list[dict[str, Any]], vnc_port: int | None = None) -> dict[str, Any] | None: - """Choose the page target a human is most likely driving. - - Skips workers/service targets, the built-in tab-search UI, and the local - VNC viewer page itself (so `cuttle view`/`login` never grab their own UI). - """ - viewer_prefix = f"http://127.0.0.1:{vnc_port}/" if vnc_port else None - pages = [t for t in targets if t.get("type") == "page"] - for t in pages: - url = t.get("url", "") - if url.startswith("chrome://"): - continue - if viewer_prefix and url.startswith(viewer_prefix): - continue - return t - return pages[0] if pages else None - - -class CDPSession: - """A single WebSocket connection to one CDP target, with id-matched calls.""" - - def __init__(self, ws: websockets.ClientConnection) -> None: - self._ws = ws - self._next_id = 0 - - async def call(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]: - self._next_id += 1 - msg_id = self._next_id - await self._ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}})) - while True: - raw = await self._ws.recv() - msg = json.loads(raw) - if msg.get("id") == msg_id: - if "error" in msg: - raise RuntimeError(f"{method}: {msg['error'].get('message', msg['error'])}") - return msg.get("result", {}) - - -async def navigate(cdp_port: int, url: str, vnc_port: int | None = None) -> str: - """Navigate the active page to `url`; return its title (best-effort).""" - target = pick_page(list_targets(cdp_port), vnc_port) - if not target or not target.get("webSocketDebuggerUrl"): - raise RuntimeError("no page target found to navigate") - async with websockets.connect(target["webSocketDebuggerUrl"], max_size=None) as ws: - session = CDPSession(ws) - await session.call("Page.navigate", {"url": url}) - try: - res = await session.call( - "Runtime.evaluate", - {"expression": "document.title", "returnByValue": True}, - ) - return str(res.get("result", {}).get("value", "") or "") - except RuntimeError: - return "" diff --git a/cuttle/cli.py b/cuttle/cli.py deleted file mode 100644 index e005220..0000000 --- a/cuttle/cli.py +++ /dev/null @@ -1,531 +0,0 @@ -"""The `cuttle` CLI: manage a local stealth-browser container and its viewer. - -The default viewer is the offscreen VNC stack baked into the image (headed -Chrome on Xvnc, browser-only noVNC page) - the clean, working path. `cuttle -view` is an experimental alternative that streams the page over CDP screencast -instead; see cuttle.view. - -Design: this is a thin host-side orchestrator. `up` shells out to `docker run` -with the VNC env/ports; `login` and `status` talk to the container's CDP over -HTTP/WebSocket. It holds no long-lived state - the container is the state. -""" - -from __future__ import annotations - -import argparse -import asyncio -import concurrent.futures -import json -import os -import shutil -import subprocess -import sys -import time -import urllib.error -import urllib.request -import webbrowser -from importlib import metadata, resources -from typing import NamedTuple, NoReturn - -DEFAULT_NAME = "cuttle" -IMAGE_REPO = "ghcr.io/glim-sh/cuttle" -DEFAULT_CDP_PORT = 9222 -DEFAULT_VNC_PORT = 6080 - - -class _Driver(NamedTuple): - """A CDP driver CLI cuttle knows how to route agents to. - - cuttle never bakes in driver documentation - each driver self-documents at - runtime (`docs`), so instructions always match the installed version. The - briefing only carries the attach incantation and where the docs live. - """ - - name: str - attach: str # .format(cdp=, port=) - docs: str - install: str - # None = never probe: browser-use treats unknown argv as harness input and - # would launch its daemon from a mere version check. - version_args: tuple[str, ...] | None - - -# Briefing order IS the fallback order: first installed entry is the default. -_DRIVERS = ( - _Driver( - name="agent-browser", - # Per-command --cdp, never `connect`: on macOS `connect` can relaunch its - # own local Chrome instead of attaching ("[agent-browser] relaunched - # browser") and then silently drives a logged-out browser that is not cuttle. - attach="agent-browser --cdp {port} # --cdp on EVERY command; never `connect`", - docs="agent-browser skills get core --full", - install="npm install -g agent-browser", - version_args=("--version",), - ), - _Driver( - name="browser-use", - attach="BU_CDP_URL={cdp} browser-use <<'PY' ... PY", - docs="browser-use skill show", - install="uv tool install browser-use", - version_args=None, - ), - _Driver( - name="playwright-cli", - attach="playwright-cli attach --cdp={cdp}", - docs="playwright-cli --help # its 'Agent skill:' line -> full SKILL.md + references/", - install="npm install -g @playwright/cli", - version_args=("--version",), - ), -) - - -def _cuttle_version() -> str: - try: - return metadata.version("cuttle-browser") - except metadata.PackageNotFoundError: - return "dev" - - -def _driver_version(name: str, exe: str, version_args: tuple[str, ...]) -> str | None: - try: - r = subprocess.run( - [exe, *version_args], - text=True, - capture_output=True, - timeout=5, - env={**os.environ, "NO_UPDATE_NOTIFIER": "1"}, - ) - except (OSError, subprocess.SubprocessError): - return None - first = ((r.stdout or r.stderr).strip().splitlines() or [""])[0] - if r.returncode != 0 or not first: - return None - # Some drivers echo their own name ("agent-browser 0.31.1"); the briefing - # already prints the name, so keep just the version. - return first.removeprefix(name).strip()[:40] or None - - -def _detect_drivers() -> list[tuple[_Driver, str | None]]: - """Return (driver, version|None) for each installed driver, briefing order. - - Version probes run in parallel so the briefing stays fast; a probe failure - degrades to a versionless line, never an error. - """ - installed = [(d, shutil.which(d.name)) for d in _DRIVERS] - installed = [(d, exe) for d, exe in installed if exe] - if not installed: - return [] - with concurrent.futures.ThreadPoolExecutor(max_workers=len(installed)) as pool: - futures = { - d.name: pool.submit(_driver_version, d.name, exe, d.version_args) - for d, exe in installed - if d.version_args is not None - } - return [(d, futures[d.name].result() if d.name in futures else None) for d, _ in installed] - - -def _docker() -> str: - exe = shutil.which("docker") - if not exe: - _die("docker not found on PATH - install Docker (or OrbStack) first.") - return exe - - -def _die(msg: str, code: int = 1) -> NoReturn: - print(f"cuttle: {msg}", file=sys.stderr) - raise SystemExit(code) - - -def _run(args: list[str], *, capture: bool = False) -> subprocess.CompletedProcess[str]: - return subprocess.run(args, text=True, capture_output=capture, check=False) - - -def _default_image() -> str: - """The published image tag matching this CLI's version, so it never drives a - cuttleserve it wasn't shipped with. An uninstalled checkout reports "dev" - (no such tag), so fall back to `latest`.""" - v = _cuttle_version() - return f"{IMAGE_REPO}:{v if v != 'dev' else 'latest'}" - - -def _inspect(name: str, field: str) -> str | None: - """A single `docker inspect -f {{field}}` value, or None if the container - doesn't exist / the field is empty.""" - r = _run([_docker(), "inspect", "-f", f"{{{{{field}}}}}", name], capture=True) - return r.stdout.strip() or None if r.returncode == 0 else None - - -def _container_state(name: str) -> str | None: - """Return 'running', 'exited', ... or None if the container doesn't exist.""" - return _inspect(name, ".State.Status") - - -def _container_image(name: str) -> str | None: - return _inspect(name, ".Config.Image") - - -def _container_ports(name: str) -> str: - """The container's real host<-container port bindings, verbatim from docker - (e.g. '9222/tcp -> 127.0.0.1:9222'). Empty if it has none.""" - r = _run([_docker(), "port", name], capture=True) - return r.stdout.strip() if r.returncode == 0 else "" - - -def _print_logs_tail(name: str, lines: int = 20) -> None: - r = _run([_docker(), "logs", "--tail", str(lines), name], capture=True) - out = (r.stdout + r.stderr).strip() - if out: - print(f" last {lines} log lines:") - for line in out.splitlines(): - print(f" {line}") - - -def _cdp_ready(port: int, timeout: float = 0.5) -> dict | None: - try: - with urllib.request.urlopen( - f"http://127.0.0.1:{port}/json/version", timeout=timeout - ) as resp: - return json.loads(resp.read()) - except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError): - return None - - -def _wait_cdp(port: int, timeout: float = 30.0) -> dict | None: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - v = _cdp_ready(port) - if v: - return v - time.sleep(0.5) - return None - - -def _urls(cdp_port: int, vnc_port: int) -> tuple[str, str]: - return (f"http://127.0.0.1:{cdp_port}", f"http://127.0.0.1:{vnc_port}/") - - -def cmd_up(args: argparse.Namespace) -> int: - """Start the container with VNC viewing on. Idempotent, and profile- - preserving: a stopped container is restarted (its browser profile lives in - the container's filesystem, so logins persist across down/up) rather than - recreated. `--recreate` forces a clean container + fresh profile.""" - state = _container_state(args.name) - if args.recreate and state is not None: - _run([_docker(), "rm", "-f", args.name], capture=True) - state = None - - # A container that never ran cleanly is a failed-run zombie (state "created" - # from a `docker run` that died at network setup, or "dead") - it has no live - # host port binding to restart into. Remove it and fall through to a fresh - # run. Only "exited" is safe to restart: that is a clean `cuttle down`. - if state is not None and state not in ("running", "exited"): - _run([_docker(), "rm", "-f", args.name], capture=True) - state = None - - # The profile policy and the image are baked in on the `docker run` path, so - # an existing container keeps whatever it was created with. Say so rather - # than appearing to honour a flag we are about to ignore. - if state is not None and args.keep_profile is not None: - print( - f"cuttle: --keep-profile is fixed when the container is created; " - f"'{args.name}' keeps its original setting (use --recreate to change it)", - file=sys.stderr, - ) - if state is not None and args.image: - print( - f"cuttle: --image is fixed when the container is created; '{args.name}' keeps " - f"the image it was created with (use --recreate to change it)", - file=sys.stderr, - ) - - if state == "running": - v = _cdp_ready(args.cdp_port) - if not v: - _die( - f"container '{args.name}' is running but CDP on :{args.cdp_port} is not " - f"answering - run `cuttle status` to triage, then `cuttle down` and retry." - ) - _print_briefing(args, "already running", browser=v.get("Browser")) - return 0 - - if state is not None: - # Exists but stopped - restart it, keeping the profile (logins). - r = _run([_docker(), "start", args.name], capture=True) - if r.returncode != 0: - _die(f"docker start failed:\n{r.stderr.strip()}") - v = _wait_cdp(args.cdp_port) - if not v: - _die( - f"container restarted but CDP on :{args.cdp_port} never came up - " - f"run `cuttle status` to triage (a port mismatch is the usual cause)." - ) - _print_briefing(args, "restarted", browser=v.get("Browser")) - return 0 - - args.image = args.image or _default_image() - - docker_args = [ - _docker(), - "run", - "-d", - "--name", - args.name, - "-p", - f"127.0.0.1:{args.cdp_port}:9222", - "--shm-size=2g", - ] - if not args.no_vnc: - docker_args += ["-p", f"127.0.0.1:{args.vnc_port}:6080", "-e", "CUTTLE_VNC=1"] - # cuttleserve defaults to port 9222 and auto-binds 0.0.0.0 in a container, - # so pass neither - it only accepts the `=` form and has no --host flag; - # the space forms would leak onto Chrome's argv as junk positional tabs. - docker_args += [args.image, "cuttleserve"] - if args.keep_profile is not False: - docker_args.append("--keep-profile") - - r = _run(docker_args, capture=True) - if r.returncode != 0: - # A run that fails at network setup (e.g. host port taken) still leaves a - # half-created container behind - remove it so it is not mistaken for a - # restartable container on the next `cuttle up`. - _run([_docker(), "rm", "-f", args.name], capture=True) - _die(f"docker run failed:\n{r.stderr.strip()}") - - v = _wait_cdp(args.cdp_port) - if not v: - _die( - f"container started but CDP on :{args.cdp_port} never came up - " - f"run `cuttle status` to triage." - ) - _print_briefing(args, "ready", browser=v.get("Browser"), show_image=True) - return 0 - - -def _print_briefing( - args: argparse.Namespace, - verb: str, - *, - browser: str | None = None, - show_image: bool = False, -) -> None: - """The briefing: the single dynamic source of truth an agent needs to drive - cuttle. Live state + installed drivers with attach lines and their own - self-doc commands. cuttle carries no driver docs of its own - see _Driver.""" - version = _cuttle_version() - cdp, viewer = _urls(args.cdp_port, args.vnc_port) - tail = f", image {args.image}" if show_image else "" - engine = f" ({browser})" if browser else "" - print(f"cuttle {verb} (container '{args.name}'{tail}) cuttle-browser {version}") - print(f" CDP {cdp}{engine}") - if not args.no_vnc: - print(f" viewer {viewer}") - print() - print("Attach to THIS browser over CDP. NEVER launch your own browser or create a") - print("new profile/context: logins live in this one and persist across down/up.") - print() - drivers = _detect_drivers() - if drivers: - print("drivers (listed in priority order; the first is the default):") - for d, ver in drivers: - print(f" {d.name}" + (f" {ver}" if ver else "")) - print(f" attach {d.attach.format(cdp=cdp, port=args.cdp_port)}") - print(f" docs {d.docs}") - for d in _DRIVERS: - if not any(inst.name == d.name for inst, _ in drivers): - print(f" {d.name} not installed (install: {d.install})") - print("routing: use the first driver listed above unless the user names another") - print(" (bu / bu-cli / browseruse = browser-use). If the named driver is not") - print(" installed, use the first listed instead and tell the user you fell back.") - print("docs: fetch each driver's own instructions with the `docs` command above -") - print(" they match the installed version; do not rely on memory or stale copies.") - else: - print("drivers: none installed. STOP and ask the user what to install -") - print(" default: all three; minimal: just agent-browser (the default driver).") - for d in _DRIVERS: - print(f" {d.install}") - print(" (drivers attach to cuttle's browser - skip their own browser downloads)") - if not args.no_vnc: - print("login walls / captcha: `cuttle login `, then hand the user the viewer") - print(" link to sign in or solve it - the CDP session stays logged in.") - print("full cuttle guide: `cuttle skill` (skip if the cuttle skill is already loaded") - print(f" in your context and its version matches {version}; rerun it if not)") - - -def cmd_status(args: argparse.Namespace) -> int: - """Single diagnostic surface: the briefing when healthy, and when not, the - real image, real host port bindings, and a log tail - so triage never needs - a raw docker command.""" - state = _container_state(args.name) - if state is None: - print(f"cuttle: no container '{args.name}' (run `cuttle up`)") - return 1 - v = _cdp_ready(args.cdp_port) - if state == "running" and v: - # Healthy: the CDP/viewer URLs are the container's real bindings (CDP just - # answered on them); add the real image so status surfaces drift either way. - _print_briefing(args, "running", browser=v.get("Browser")) - if image := _container_image(args.name): - print(f" image {image}") - return 0 - - # Unhealthy: report what is actually there, not what was requested. The most - # common cause is a CDP port that does not match the container's real binding. - cdp, viewer = _urls(args.cdp_port, args.vnc_port) - print(f"container '{args.name}': {state}") - print(f" CDP {cdp} (not answering)" if not v else f" CDP {cdp}") - if not args.no_vnc: - print(f" viewer {viewer}") - if image := _container_image(args.name): - print(f" image {image}") - if ports := _container_ports(args.name): - print(" actual port bindings (start `up` with THESE ports, do not --recreate):") - for line in ports.splitlines(): - print(f" {line}") - _print_logs_tail(args.name) - print(" fix: `cuttle down && cuttle up` (keeps the profile), or") - print(" `cuttle up --recreate` to rebuild from scratch (discards the profile).") - return 1 - - -def cmd_down(args: argparse.Namespace) -> int: - """Stop the container gracefully (SIGTERM), which lets cuttleserve shut - Chrome down cleanly - so it records a clean exit and never crash-restores - junk tabs. The container (and its logged-in profile) is kept for the next - `cuttle up`; pass --purge to remove it and discard the profile.""" - state = _container_state(args.name) - if state is None: - print(f"cuttle: no container '{args.name}'") - return 0 - if state == "running": - # -t 15 > cuttleserve's 5s Chrome-drain, so the clean exit completes. - r = _run([_docker(), "stop", "-t", "15", args.name], capture=True) - if r.returncode != 0: - _die(f"docker stop failed:\n{r.stderr.strip()}") - if args.purge: - r = _run([_docker(), "rm", "-f", args.name], capture=True) - if r.returncode != 0: - _die(f"docker rm failed:\n{r.stderr.strip()}") - print(f"cuttle: removed container '{args.name}' (profile discarded)") - else: - print(f"cuttle: stopped container '{args.name}' (profile kept; `cuttle up` to resume)") - return 0 - - -def cmd_login(args: argparse.Namespace) -> int: - """Navigate the browser to a URL and hand back the viewer link.""" - if not _cdp_ready(args.cdp_port): - _die(f"CDP on :{args.cdp_port} not answering - run `cuttle up` first.") - from cuttle.cdp import navigate - - vnc_port = None if args.no_vnc else args.vnc_port - try: - title = asyncio.run(navigate(args.cdp_port, args.url, vnc_port)) - except Exception as exc: # noqa: BLE001 - surface any CDP failure to the user - _die(f"navigation failed: {exc}") - _, viewer = _urls(args.cdp_port, args.vnc_port) - print(f"navigated to {args.url}" + (f" ({title})" if title else "")) - if not args.no_vnc: - print(f"open the viewer to sign in: {viewer}") - if not args.no_open: - webbrowser.open(viewer) - return 0 - - -def cmd_view(args: argparse.Namespace) -> int: - """EXPERIMENTAL: serve a CDP-screencast viewer instead of VNC.""" - if not _cdp_ready(args.cdp_port): - _die(f"CDP on :{args.cdp_port} not answering - run `cuttle up` first.") - from cuttle.view import serve - - print(f"cuttle view (experimental): CDP :{args.cdp_port} -> http://{args.listen}/") - if not args.no_open: - webbrowser.open(f"http://{args.listen}/") - try: - asyncio.run(serve(args.cdp_port, args.listen, target_url=args.url)) - except KeyboardInterrupt: - pass - return 0 - - -def cmd_skill(args: argparse.Namespace) -> int: - """Print SKILL.md (the agent usage guide) to stdout, so `cuttle skill` hands - an agent the how-to-drive-cuttle doc. SKILL.md is bundled as package data - (see pyproject package-data), so this works from a plain `pip install`.""" - text = resources.files("cuttle").joinpath("SKILL.md").read_text(encoding="utf-8") - print(text, end="") - return 0 - - -def _add_common(p: argparse.ArgumentParser) -> None: - p.add_argument("--name", default=DEFAULT_NAME, help=f"container name (default {DEFAULT_NAME})") - p.add_argument("--cdp-port", type=int, default=DEFAULT_CDP_PORT, help="host CDP port") - p.add_argument("--vnc-port", type=int, default=DEFAULT_VNC_PORT, help="host VNC viewer port") - p.add_argument("--no-vnc", action="store_true", help="run without the VNC viewer") - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="cuttle", - description="Manage a local stealth-browser container and its viewer.", - ) - sub = parser.add_subparsers(dest="cmd", required=True) - - up = sub.add_parser("up", help="start the container (idempotent) with VNC viewing") - _add_common(up) - # Both default=None so the restart path can tell "not passed" from an explicit - # choice and warn that the setting is fixed at container creation. - up.add_argument( - "--image", - default=None, - help=f"image (default {_default_image()}; use `--image cuttle:local` for a local build)", - ) - up.add_argument( - "--keep-profile", - action=argparse.BooleanOptionalAction, - default=None, - help="persist the browser profile across restarts (default on)", - ) - up.add_argument( - "--recreate", - action="store_true", - help="destroy any existing container and start fresh (discards the profile)", - ) - up.set_defaults(func=cmd_up) - - login = sub.add_parser("login", help="navigate to a URL and open the viewer to sign in") - _add_common(login) - login.add_argument("url", help="URL to open in the browser") - login.add_argument("--no-open", action="store_true", help="print the viewer URL, don't open it") - login.set_defaults(func=cmd_login) - - st = sub.add_parser("status", help="show container + CDP state") - _add_common(st) - st.set_defaults(func=cmd_status) - - down = sub.add_parser("down", help="stop the container gracefully (keeps the profile)") - _add_common(down) - down.add_argument( - "--purge", action="store_true", help="also remove the container and discard the profile" - ) - down.set_defaults(func=cmd_down) - - view = sub.add_parser("view", help="EXPERIMENTAL: CDP-screencast viewer instead of VNC") - _add_common(view) - view.add_argument("--listen", default="127.0.0.1:6090", help="host:port for the viewer") - view.add_argument("--url", default=None, help="navigate here before viewing") - view.add_argument("--no-open", action="store_true", help="don't auto-open the viewer") - view.set_defaults(func=cmd_view) - - sk = sub.add_parser("skill", help="print the SKILL.md agent usage guide to stdout") - sk.set_defaults(func=cmd_skill) - - return parser - - -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) - return args.func(args) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/cuttle/view.py b/cuttle/view.py deleted file mode 100644 index c232549..0000000 --- a/cuttle/view.py +++ /dev/null @@ -1,214 +0,0 @@ -"""EXPERIMENTAL: a browser-only viewer that streams the page over CDP -screencast instead of VNC. - -Why it exists: the VNC path (headed Chrome on Xvnc + noVNC) is the working -default, but it carries KasmVNC's forked RFB dialect. CDP screencast is a -dialect-free alternative - Page.startScreencast for frames, Input.dispatch* -for mouse/keyboard, all over the same CDP the agent already uses. Input goes -in as synthesized trusted events (a stealth win, same reason we drive with -compositor-level events elsewhere). - -Known limits (why this is experimental, not the default): -- Screencast captures the page VIEWPORT only - not browser chrome, not native - OS dialogs. Fine for login forms and captchas (page content). -- OAuth sign-in popups are a SEPARATE CDP target; this MVP views one page - target at a time and does not yet auto-attach to popups. The VNC path shows - the whole display and handles popups for free - keep it for those flows. -""" - -from __future__ import annotations - -import asyncio -import json -from typing import Any - -import websockets -from aiohttp import WSMsgType, web - -from cuttle.cdp import list_targets, pick_page - - -class Bridge: - """Fans one CDP screencast out to N browser clients and their input back.""" - - def __init__(self) -> None: - self.clients: set[web.WebSocketResponse] = set() - self.out: asyncio.Queue[dict[str, Any]] = asyncio.Queue() - self.meta: dict[str, Any] = {"deviceWidth": 0, "deviceHeight": 0} - - def send_cdp(self, method: str, params: dict[str, Any] | None = None) -> None: - self.out.put_nowait({"method": method, "params": params or {}}) - - async def broadcast(self, payload: dict[str, Any]) -> None: - text = json.dumps(payload) - targets = list(self.clients) - results = await asyncio.gather( - *(ws.send_str(text) for ws in targets), return_exceptions=True - ) - for ws, result in zip(targets, results, strict=True): - if isinstance(result, Exception): - self.clients.discard(ws) - - def input_to_cdp(self, msg: dict[str, Any]) -> None: - """Translate a viewer input event into a CDP Input.* command.""" - kind = msg.get("kind") - if kind == "mouse": - dw = self.meta.get("deviceWidth") or 0 - dh = self.meta.get("deviceHeight") or 0 - x = float(msg.get("fx", 0)) * dw - y = float(msg.get("fy", 0)) * dh - params: dict[str, Any] = { - "type": msg["type"], # mousePressed | mouseReleased | mouseMoved - "x": x, - "y": y, - "button": msg.get("button", "none"), - "buttons": msg.get("buttons", 0), - "clickCount": msg.get("clickCount", 0), - "modifiers": msg.get("modifiers", 0), - } - if msg["type"] == "mouseWheel": - params["deltaX"] = msg.get("deltaX", 0) - params["deltaY"] = msg.get("deltaY", 0) - self.send_cdp("Input.dispatchMouseEvent", params) - elif kind == "key": - params = { - "type": msg["type"], # keyDown | keyUp | char - "modifiers": msg.get("modifiers", 0), - "key": msg.get("key", ""), - "code": msg.get("code", ""), - "windowsVirtualKeyCode": msg.get("keyCode", 0), - } - if msg.get("text"): - params["text"] = msg["text"] - self.send_cdp("Input.dispatchKeyEvent", params) - - -async def _cdp_reader(ws: websockets.ClientConnection, bridge: Bridge) -> None: - async for raw in ws: - msg = json.loads(raw) - if msg.get("method") == "Page.screencastFrame": - p = msg.get("params", {}) - bridge.meta = p.get("metadata", bridge.meta) - # Ack before the fanout: Chrome withholds the next frame until the - # ack, so a slow viewer must not gate the stream for the others. - bridge.send_cdp("Page.screencastFrameAck", {"sessionId": p.get("sessionId")}) - await bridge.broadcast({"type": "frame", "data": p.get("data", "")}) - - -async def _cdp_writer(ws: websockets.ClientConnection, bridge: Bridge) -> None: - next_id = 0 - while True: - cmd = await bridge.out.get() - next_id += 1 - await ws.send(json.dumps({"id": next_id, **cmd})) - - -async def _cdp_pump(ws_url: str, bridge: Bridge, target_url: str | None) -> None: - async with websockets.connect(ws_url, max_size=None) as ws: - writer = asyncio.create_task(_cdp_writer(ws, bridge)) - bridge.send_cdp("Page.enable") - if target_url: - bridge.send_cdp("Page.navigate", {"url": target_url}) - bridge.send_cdp( - "Page.startScreencast", - {"format": "jpeg", "quality": 70, "everyNthFrame": 1}, - ) - try: - await _cdp_reader(ws, bridge) - finally: - writer.cancel() - - -async def _index(_req: web.Request) -> web.Response: - return web.Response(text=_PAGE, content_type="text/html") - - -async def _client_ws(req: web.Request) -> web.WebSocketResponse: - bridge: Bridge = req.app["bridge"] - ws = web.WebSocketResponse() - await ws.prepare(req) - bridge.clients.add(ws) - try: - async for msg in ws: - if msg.type == WSMsgType.TEXT: - bridge.input_to_cdp(json.loads(msg.data)) - elif msg.type == WSMsgType.ERROR: - break - finally: - bridge.clients.discard(ws) - return ws - - -async def serve(cdp_port: int, listen: str, target_url: str | None = None) -> None: - host, _, port = listen.partition(":") - target = pick_page(list_targets(cdp_port)) - if not target or not target.get("webSocketDebuggerUrl"): - raise RuntimeError("no page target found to view") - - bridge = Bridge() - app = web.Application() - app["bridge"] = bridge - app.router.add_get("/", _index) - app.router.add_get("/ws", _client_ws) - - runner = web.AppRunner(app) - await runner.setup() - site = web.TCPSite(runner, host or "127.0.0.1", int(port)) - await site.start() - try: - await _cdp_pump(target["webSocketDebuggerUrl"], bridge, target_url) - finally: - await runner.cleanup() - - -_PAGE = """ -cuttle view - - -
connecting...
-""" diff --git a/docs/BUILD-FROM-SOURCE.md b/docs/BUILD-FROM-SOURCE.md deleted file mode 100644 index 9082301..0000000 --- a/docs/BUILD-FROM-SOURCE.md +++ /dev/null @@ -1,24 +0,0 @@ -# Building a stealth-Chromium binary from source (break-glass) - -cuttle consumes clark/clearcote as pinned prebuilt binaries. Building Chromium -from source is the single highest-maintenance job in the stack and is NOT part -of the normal pipeline. This is insurance for the case where both forks stall -on a Chrome major you need before either ships a prebuilt. - -Both forks are open and buildable: - -- **clark-browser** ships a reproducible build chain: `fetch-source.sh` -> - `apply-patches.sh` -> `build.sh` (plus a `Dockerfile.linux`). The stealth - behavior is a `--fingerprint-*` patch series applied on top of an - ungoogled-chromium checkout. -- **clearcote-browser**'s engine is open and buildable on the same lines. - -Machine requirements (per build): ~80 GB free disk, 32 GB+ RAM, roughly -4-12 hours of build time on a fast machine. - -Caveat: when the Chrome base moves a major version, the `--fingerprint-*` -patch series usually needs a rebase against the new source tree. Budget time -for patch conflicts, not just compile time. - -Prefer waiting for a fork prebuilt. Only build from source when there is a hard -deadline and no released binary covers the Chrome version you need. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 627c423..e9efea0 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -1,62 +1,47 @@ # Releasing cuttle PR-merge-driven via [release-please](https://github.com/googleapis/release-please), -fully automated once the release PR is merged. The version comes from conventional -commits; the changelog stays git-cliff (`.github/cliff.toml`), mirroring pond's format. +fully automated once the release PR is merged. The version and changelog come from +the conventional commits on `main`. ## Cutting a release 1. Land conventional commits on `main` (`feat:`, `fix:`, `chore:` ... - they become the changelog and drive the semver bump). release-please keeps a - `chore(main): release X.Y.Z` PR open and up to date, bumping `pyproject.toml` - and `SKILL.md`'s frontmatter version. Preview the pending changelog with - `just release-preview`. -2. Run the gates: `just smoke`, then the real-amd64 deployment check - (`docs/UPGRADE.md`). CI has no test gate by design. -3. **Merge the release PR.** That is the release: release-please tags `vX.Y.Z` - and opens the GitHub release, then the gated jobs in `release.yml` publish. - -To force a specific version (e.g. a minor bump the commits would not derive), add -a commit with a `Release-As: X.Y.Z` footer before merging; otherwise the version -is derived automatically (non-breaking commits -> patch, `feat!`/`fix!` -> minor -pre-1.0). - -## What the merge triggers (`.github/workflows/release.yml`, one run) - -- `release-please` - tags `vX.Y.Z`, opens the GitHub release, emits the outputs - the jobs below gate on (`release_created`, `version`, `tag_name`) -- `dist` - `uv build` (sdist + wheel) -- `pypi` - `uv publish` via PyPI trusted publishing (OIDC, no token) -- `image` - linux/amd64 Docker image to `ghcr.io/glim-sh/cuttle` (tags - `X.Y.Z`, `X.Y`, `latest`, `sha-...`) -- `finalize` - puts git-cliff notes + dist assets on the release, and commits the - regenerated `CHANGELOG.md` and `uv.lock` back to `main` (release-please skips - both: `skip-changelog`, and it has no uv support) -- `homebrew` - renders `Formula/cuttle.rb` from `uv.lock` (pinned sdist - resources) and pushes it to `tenequm/homebrew-tap`; the formula fetches the - sdist from the GitHub release - -The nix flake (`flake.nix`) builds from source at any rev, so it needs no -per-release step. + `chore(main): release X.Y.Z` PR open and up to date, bumping the version in + `internal/cli/SKILL.md`'s frontmatter and `CHANGELOG.md`. +2. Run the gates: `go run ./test/smoke` against the built image, then the + real-amd64 deployment check (`docs/UPGRADE.md`). +3. **Merge the release PR.** That is the release: release-please tags `vX.Y.Z` and + opens the GitHub release, then the gated steps in the `release` job of + `.github/workflows/ci.yml` publish (GoReleaser binaries + Homebrew cask, and + the GHCR image) in the same run. + +To force a specific version, add a commit with a `Release-As: X.Y.Z` footer before +merging; otherwise the version is derived automatically (non-breaking commits -> +patch, `feat!`/`fix!` -> minor pre-1.0). + +## What the merge triggers (one `ci.yml` run) + +- **release-please** - tags `vX.Y.Z`, opens the GitHub release, emits the + `release_created` / `version` outputs the steps below gate on. +- **GoReleaser** (`ops/config/goreleaser.yaml`) - cross-builds the `cuttle` CLI + (linux/darwin x amd64/arm64), appends the archives + checksums to the release, + and pushes the Homebrew cask to `tenequm/homebrew-tap`. +- **image** - the linux/amd64 Docker image to `ghcr.io/glim-sh/cuttle` (tags + `X.Y.Z`, `X.Y`, `latest`, `sha-...`). ## Install channels -- `uv tool install cuttle-browser` (or `pip install cuttle-browser`) - `brew install tenequm/tap/cuttle` -- `nix run github:glim-sh/cuttle` +- `go install github.com/glim-sh/cuttle/cmd/cuttle@latest` - `docker pull ghcr.io/glim-sh/cuttle:` ## One-time setup (repo admin) -- **PyPI trusted publisher**: on pypi.org, add a pending publisher for project - `cuttle-browser`: owner `glim-sh`, repo `cuttle`, workflow `release.yml`, - environment `pypi`. No API token needed; the `pypi` GitHub environment is - auto-created on first run. -- **`GH_RELEASE_TOKEN` secret** on `glim-sh/cuttle`: fine-grained PAT with - Contents read/write on `tenequm/homebrew-tap` (used only by the `homebrew` - job). release-please itself runs on the default `github.token`. +- **`GH_RELEASE_TOKEN` secret** on the repo: a fine-grained PAT with Contents + read/write on `tenequm/homebrew-tap` (used only by GoReleaser to push the cask). + release-please and the image push run on the default `github.token`. - **Allow GitHub Actions to create PRs**: repo Settings -> Actions -> General -> "Allow GitHub Actions to create and approve pull requests" must be on, or release-please cannot open the release PR. -- Optional: `nix flake lock` once and commit `flake.lock` to pin nixpkgs for - flake consumers. diff --git a/docs/STEALTH-VERIFICATION.md b/docs/STEALTH-VERIFICATION.md index 14bc4bf..b11c6db 100644 --- a/docs/STEALTH-VERIFICATION.md +++ b/docs/STEALTH-VERIFICATION.md @@ -2,9 +2,9 @@ How to confirm a running cuttle seed presents a coherent, healthy browser identity - and how to read the noise Chrome prints on the way there. This is the -"is the browser actually OK?" checklist; `test/harness.py` automates the -per-seed isolation and coherence checks, this doc covers what a good identity -looks like and the gotchas that look alarming but aren't. +"is the browser actually OK?" checklist; `test/smoke` (`go run ./test/smoke`) +automates the per-seed isolation and coherence checks, this doc covers what a +good identity looks like and the gotchas that look alarming but aren't. ## Probe a live seed @@ -64,8 +64,7 @@ string is a well-known automation tell. The fork instead spoofs a real GPU strin on top of whatever renders underneath, so the coherent ANGLE/Direct3D11 identity is what a detector sees. `--ignore-gpu-blocklist` (already in the base args) is what lets WebGL work at all under software rendering; the fork's patches make it -*look* real. Upstream `cloakserve` reaches the same conclusion and actively -suppresses this flag for the same reason. +*look* real. The stealth-Chromium forks suppress this flag for the same reason. ## Challenge cold-clear depends on the exit IP, not the fingerprint diff --git a/THIRD-PARTY.md b/docs/THIRD-PARTY.md similarity index 62% rename from THIRD-PARTY.md rename to docs/THIRD-PARTY.md index 35bce54..b4a876e 100644 --- a/THIRD-PARTY.md +++ b/docs/THIRD-PARTY.md @@ -1,45 +1,9 @@ # Third-party licenses -cuttle vendors and redistributes the following third-party software. Their -license terms are reproduced in full below. - ---- - -## cloakbrowser / cloakserve (MIT) - -`bin/cuttleserve` is a patched derivative of CloakHQ's `cloakserve`, and the -`cuttle/` Python package vendors a trimmed subset of CloakHQ's `cloakbrowser` -wrapper (the CDP argument-builders in `browser.py`, plus `config.py`, -`geoip.py`, and a trimmed `download.py`). Vendored from cloakbrowser v0.4.9 -(commit `045219b488e79d9fa091d1e751b98e0a7449afd1`); see `docs/UPSTREAM.md`. - -CloakBrowser's proprietary compiled Chromium binary is NOT used and NOT -redistributed. CloakHQ's `BINARY-LICENSE.md` applies only to that binary and -explicitly does not apply to the MIT wrapper source vendored here. - -``` -MIT License - -Copyright (c) 2026 CloakHQ - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` +cuttle redistributes the third-party software below; their license terms are +reproduced in full. Portions of `internal/fingerprint` and `cuttle serve` also +derive from the MIT-licensed `cloakbrowser`/`cloakserve`, used under the MIT +license; no third-party source or binary from them is redistributed. --- @@ -47,7 +11,7 @@ SOFTWARE. Prebuilt stealth-Chromium binary (`/opt/clark/chrome`) baked into the published image. Downloaded and sha256-verified at build time from the -project's GitHub releases; see the `Dockerfile`. +project's GitHub releases; see `ops/docker/Dockerfile`. ``` MIT License @@ -83,7 +47,7 @@ project for those notices. Prebuilt stealth-Chromium binary (`/opt/clearcote/chrome`) baked into the published image as a fallback engine. Downloaded and sha256-verified at build -time from the project's GitHub releases; see the `Dockerfile`. +time from the project's GitHub releases; see `ops/docker/Dockerfile`. ``` BSD 3-Clause License @@ -115,3 +79,16 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ``` + +--- + +## Windows font pack (OFL 1.1) + +The metric-compatible free fonts under `ops/docker/winfonts/` (Liberation, +Carlito, Caladea) are redistributed with their `name` table set to the +corresponding Windows family; see `ops/docker/winfonts/README.md` for the +mapping and provenance. All are licensed under the SIL Open Font License 1.1. No +Microsoft font software is included. +``` +SIL OPEN FONT LICENSE Version 1.1 - full text at https://openfontlicense.org +``` diff --git a/docs/UPGRADE.md b/docs/UPGRADE.md index fa1b88c..80bbbdb 100644 --- a/docs/UPGRADE.md +++ b/docs/UPGRADE.md @@ -5,32 +5,39 @@ one decision and one test. To move to a new Chrome major (or patch): 1. Pick the new clark (and/or clearcote) release for the target Chrome version. Update the `CLARK_TAG` / `CLARK_ASSET` / `CLARK_SHA256` (and clearcote's) - build args in the `Dockerfile`. Get the sha256 from the release's checksum, - or `curl -fsSL | sha256sum`. + build args in `ops/docker/Dockerfile`. Get the sha256 from the release's + checksum, or `curl -fsSL | sha256sum`. -2. If the vendored `cloakserve` upstream moved and matters, re-sync the - vendored subset: run `scripts/sync.sh`, review the diff, re-apply cuttle's - trims/patches, and bump the pinned ref in `docs/UPSTREAM.md`. +2. If the upstream stealth/CDP behavior cuttle mirrors has changed, reconcile + it in the Go port - the load-bearing patches (proxy-auth-over-CDP, the + service_worker `browserContextId` stamp, fork launch-parity flags) live in + `internal/serve/wsproxy.go`, and the fingerprint arg-building in + `internal/fingerprint/args.go`. Any change to argv/proxy/geoip output must be + a reviewed `internal/fingerprint/testdata/golden.json` diff (regenerate with + `just parity-golden`). 3. Confirm the flag dialect still holds - does the new binary still honor the - `--fingerprint-*` flags cuttleserve emits? - and watch for a new CDP quirk - (like Chrome 148's empty service_worker `browserContextId`). The harness + `--fingerprint-*` flags `cuttle serve` emits? - and watch for a new CDP quirk + (like Chrome 148's empty service_worker `browserContextId`). The smoke harness surfaces both. 4. Build the image and validate in two layers - they cover different risks: - - **`test/harness.py` (fast, local).** Confirms the new binary still applies - fingerprints (coherent UA/platform), isolates seeds (distinct canvas), looks - stealthy (`navigator.webdriver` falsy), and connects cleanly under cold - cycling. This is client-agnostic (raw CDP), so it CANNOT observe a new Chrome - CDP quirk that crashes a playwright client (the class of bug the - service_worker stamp fixes) - see next. + - **`test/smoke` (`go run ./test/smoke`, fast, local).** Confirms the new + binary still applies fingerprints (coherent UA/platform), isolates seeds + (distinct canvas), looks stealthy (`navigator.webdriver` falsy, real GPU via + ANGLE), and connects cleanly under cold cycling. This is client-agnostic (raw + CDP), so it CANNOT observe a new Chrome CDP quirk that crashes a playwright + client (the class of bug the service_worker stamp fixes) - see next. - **Real amd64 deployment (the gate).** Run the actual playwright-core consumer path against live sites on a real amd64 host. This is the only thing that surfaces a new playwright-crashing CDP quirk AND confirms real challenge clears. Emulated amd64-on-arm64 is fine for a smoke; real amd64 is the gate. -5. On green, publish a new `ghcr.io/glim-sh/cuttle` image (tag `v*`), then bump - the consumed digest wherever cuttle is deployed. +5. On green, publish a new `ghcr.io/glim-sh/cuttle` image (a `vX.Y.Z` release cuts + it - see `docs/RELEASING.md`), then bump the consumed digest wherever cuttle is + deployed. -If both forks stall on a needed Chrome major, see `docs/BUILD-FROM-SOURCE.md` -(break-glass only). +If both forks stall on a needed Chrome major, building the fork from source is the +break-glass path; prefer waiting for a fork prebuilt (a from-source Chromium build +needs ~80GB disk, ~32GB RAM, and hours, and the `--fingerprint-*` patches must be +rebased onto the new tag). diff --git a/docs/UPSTREAM.md b/docs/UPSTREAM.md deleted file mode 100644 index fe310bb..0000000 --- a/docs/UPSTREAM.md +++ /dev/null @@ -1,25 +0,0 @@ -# Vendored upstream - -The `vendor/cloakbrowser/` Python package and `bin/cuttleserve` derive from -CloakHQ's MIT-licensed `cloakbrowser` wrapper. This file pins the exact upstream -revision the vendored files came from so drift is a reviewable diff (see -`scripts/sync.sh`). Everything under `vendor/` is vendored; the authored `cuttle/` -package (CLI + viewer) and `test/` are ours. - -- Upstream: https://github.com/CloakHQ/cloakbrowser -- Pinned ref: `v0.4.9` (commit `045219b488e79d9fa091d1e751b98e0a7449afd1`) - -## Vendored files and their provenance - -| vendored file | upstream source | modification | -|----------------------------------|------------------------------------|--------------| -| `vendor/cloakbrowser/config.py` | `cloakbrowser/config.py` | verbatim except one unreachable Pro-license branch neutered (cuttle always runs a `CLOAKBROWSER_BINARY_PATH` override, so `.license` is never imported) | -| `vendor/cloakbrowser/geoip.py` | `cloakbrowser/geoip.py` | verbatim except three install-hint strings (upstream named the `cloakbrowser[geoip]` pip extra; neutralized - geoip2/httpx/socksio ship in cuttle's container-only `server` dep group, not the published CLI) | -| `vendor/cloakbrowser/browser.py` | `cloakbrowser/browser.py` | 3 module-level imports (license/widevine/human) replaced with local stubs; only the CDP argument-builders are used | -| `vendor/cloakbrowser/download.py`| `cloakbrowser/download.py` | trimmed to the `CLOAKBROWSER_BINARY_PATH` override path only (no Pro download / httpx / license) | -| `bin/cuttleserve` | `bin/cloakserve` (patched) | proxy-auth-over-CDP injection + service_worker browserContextId stamp + fork launch-parity flags; imports stay upstream-verbatim (`cloakbrowser.*`) | - -NOT vendored (intentionally dropped): `license.py`, `widevine.py`, `human/`, -`__main__.py`, and CloakBrowser's proprietary binary (`BINARY-LICENSE.md`). - -To re-sync after an upstream bump, run `scripts/sync.sh` and review the diff. diff --git a/docs/plans/2607-16-cuttle-go-rewrite.md b/docs/plans/2607-16-cuttle-go-rewrite.md deleted file mode 100644 index 0bdb06c..0000000 --- a/docs/plans/2607-16-cuttle-go-rewrite.md +++ /dev/null @@ -1,473 +0,0 @@ -# cuttle Go rewrite + remote backends + local-canonical profiles - -Status: PLAN (approved to build). Author handoff doc - written so a fresh agent -with no prior conversation context can build the whole thing. Read this top to -bottom before writing code. - -## 1. Goal - -Rewrite `cuttle` from Python to Go as a single, minimal, elegant codebase that: - -1. Owns the code outright - fold the in-container daemon (`bin/cuttleserve`) and - the vendored `cloakbrowser` Python into first-class authored Go. Retire the - `vendor/` + `scripts/sync.sh` upstream-vendoring relationship. -2. Turns cuttle into a universal browser-lifecycle CLI that manages the stealth - browser in one of four places, selected by a named context: **local** (docker - here), **k8s** (a Deployment in a cluster), **ssh** (docker on a remote host), - **direct** (a pre-exposed CDP/VNC URL). Same `cuttle up/down/status/login/ - connect` verbs regardless of where the browser runs. -3. Adds **local-canonical profiles**: the user's browser profiles + auth state - live on the user's machine (XDG data dir) and are injected into whatever - remote browser per session, so sensitive data resides locally at rest. Named - profiles map to cuttle "seeds". -4. Installs + configures the browser-use MCP (and other CDP drivers) pointed at - the active context's browser, so `cuttle mcp` gives an agent a working, - authenticated browser with no manual wiring. - -The end state is what the user wants for themselves first (reliable -authenticated browser access without local-browser babysitting), built cleanly -enough to become a product later. This is NOT a public paid service build. - -## 2. What we do NOT rewrite / own - -The stealth is a **patched Chromium fork** shipped as a prebuilt linux-x64 -binary, downloaded SHA-pinned in the Dockerfile and selected via -`CLOAKBROWSER_BINARY_PATH`. We keep consuming it unchanged: - -- `clark-browser` (MIT): ungoogled-chromium 148 + `--fingerprint-*` patches - (default) - `github.com/clark-labs-inc/clark-browser` -- `clearcote-browser` (BSD-3): Chromium 149 + fingerprint patches (fallback) - -We never fork Chromium. The Go rewrite covers only the thin orchestration around -that binary (CLI, CDP multiplexer, arg-building, geoip, proxy). The one ongoing -cost of owning the wrapper: we now track clark/clearcote's `--fingerprint-*` -flag surface ourselves (flags, not C++). - -## 3. Locked decisions - -| Decision | Choice | -|---|---| -| Language | Go 1.26+ (full rewrite: CLI + daemon + cloakbrowser wrapper) | -| Keep Python? | Yes, temporarily, as a reference oracle in `packages/cuttle-py/` until Go reaches parity + passes STEALTH-VERIFICATION, then delete | -| Repo layout | Monorepo: `packages/cuttle-py/` (old) + `packages/cuttle-go/` (new); per-language configs live in each package | -| Helm chart location | `ops/helm/cuttle/` (repo root, shared) | -| Config file format | TOML via `github.com/pelletier/go-toml/v2` (DECIDED - hand-edited, needs comments; comment-preserving write-back via `unstable/edit`). | -| Generated MCP config | JSON via stdlib `encoding/json` (MCP convention, no dep) | -| k8s / helm / ssh access | Shell out to `kubectl` / `helm` / `ssh` (inherit the user's kube context, ssh config, and whatever routing they provide - e.g. tailnet - with zero cuttle-specific setup). No client-go, no helm SDK. | -| Remote reachability | Native `kubectl port-forward` and `ssh -L` tunnels that map the remote browser onto `127.0.0.1:`, so all CDP/VNC code is transport-agnostic. `direct` backend uses a config URL as-is (escape hatch for any pre-exposed endpoint). | -| Profile storage default | local-canonical (storage_state checkout/checkin over CDP; remote profile dir ephemeral). Per-profile opt-in to remote-persistent for always-on/autonomous use. | -| Proxy | A server-level default set at `cuttle up` from context config, applied to the browser regardless of backend. (Today proxy is per-connection only; add a default.) geoip auto-follows the proxy. | -| idle-timeout | local only; remote browsers run without reaping. | -| CLI framework | `github.com/spf13/cobra` (help, completion, POSIX flags). Stdlib `flag` acceptable if we want fewer deps. | - -## 4. Non-goals for v1 - -- No multi-tenant/public service, no billing, no OAuth-gated hosted VNC proxy. - (Those belong to a later `glim_browser` product; this is single-owner tooling.) -- No Go rewrite of `test/harness.py` semantics beyond what parity needs. -- No client-go / helm-Go-SDK. Shelling out is the deliberate minimal choice. - -## 5. Repo restructure (do this first, as a mechanical move) - -Target layout: - -``` -cuttle/ (repo root, glim-sh/cuttle) - README.md LICENSE CHANGELOG.md - .github/workflows/ (CI for both packages) - docs/ (shared docs incl. this plan) - ops/ - helm/cuttle/ (Helm chart - NEW) - packages/ - cuttle-py/ (the ENTIRE current repo tree, moved verbatim) - cuttle/ vendor/ bin/ test/ Dockerfile pyproject.toml uv.lock - flake.nix Justfile ... (all Python-side config moves here) - cuttle-go/ (NEW authored Go) - cmd/cuttle/main.go - internal/... - Dockerfile - go.mod go.sum - .golangci.yml lefthook.yml Justfile -``` - -Rules: -- Move the existing Python tree wholesale into `packages/cuttle-py/` (git mv, one - commit, no code changes) so it keeps building and stays the oracle. -- Each language's build/lint/format config lives inside its package dir. Repo - root holds only shared things (LICENSE, top README, `.github/`, `docs/`, - `ops/`). -- Go module path during coexistence: `module github.com/glim-sh/cuttle/packages/cuttle-go`. - This makes `go install github.com/glim-sh/cuttle/packages/cuttle-go/cmd/cuttle@latest` - correct but ugly; users install via brew/release binaries where the path is - invisible, so it is low-impact. When `cuttle-py` is retired, promote `cuttle-go` - to repo root and rename the module to `github.com/glim-sh/cuttle`. - Optional: add a root `go.work` for local multi-module dev ergonomics (does not - affect remote `go install`). - -## 6. Toolchain (go-dev stack, pinned) - -Inside `packages/cuttle-go/`: -- Go 1.26+, `go mod`, tool deps tracked with `go get -tool` (Go 1.24+). -- `golangci-lint` v2.11+ (config `.golangci.yml`, template from the go-dev skill). -- `gofumpt` v0.9+ (via golangci-lint formatters, extra-rules on). -- `gotestsum` v1.13+ for test output. -- `just` task runner (`Justfile` from the go-dev skill: fmt/lint/test/check/build). -- `lefthook` pre-commit (fmt -> lint -> mod-tidy) and pre-push (test -race). -- `govulncheck` in CI. - -Use the exact `.golangci.yml`, `Justfile`, `lefthook.yml`, and GitHub Actions -templates from the `go-dev` skill. Keep `main.go` thin; business logic in -`internal/`. - -## 7. Dependencies (verified current, 2026-07-16) - -| Concern | Package | Notes | -|---|---|---| -| CDP driving (navigate, cookies, localStorage eval) | `github.com/chromedp/chromedp` + `github.com/chromedp/cdproto` | Mature, zero external deps, MIT. Connect to the already-running fork via `chromedp.RemoteAllocator` (do NOT let chromedp launch Chrome). Use `cdproto/network` (GetCookies/SetCookies), `cdproto/storage`, `cdproto/runtime` (Evaluate for localStorage) for storage_state. | -| Raw WebSocket (the multiplexer's client<->Chrome CDP proxy) | `github.com/coder/websocket` | Modern nhooyr successor, zero deps, idiomatic, context-first. Use for accepting client CDP WS and piping frames to the seed's Chrome WS. | -| GeoIP (tz/locale/exit-IP from proxy) | `github.com/oschwald/geoip2-golang/v2` | v2 API uses `netip.Addr`; `record.Location.TimeZone`, `record.Country.ISOCode`. We download+cache the GeoLite2-City.mmdb ourselves (see 8.3), the lib only reads it. Requires Go 1.25+. | -| TOML config | `github.com/pelletier/go-toml/v2` | TOML 1.1, strict mode (`DisallowUnknownFields`), and `unstable/edit` for comment-preserving write-back (future `cuttle context add/use`). | -| CLI | `github.com/spf13/cobra` | Subcommands, POSIX flags, shell completion, man pages. | -| SOCKS/HTTP proxy dialer (exit-IP echo call through the proxy) | `golang.org/x/net/proxy` | For resolving the proxy exit IP (WebRTC spoof + geoip), mirroring the Python `socksio` path. | -| Everything else | stdlib | `os/exec` (docker/kubectl/helm/ssh/Chrome), `net/http` (CDP `/json`, mmdb download, echo-IP), `encoding/json` (MCP config + CDP messages), `net/url`, `net/netip`. | - -Reference docs to consult while implementing: pkg.go.dev for each of the above; -`chromedp/examples` repo for RemoteAllocator + raw CDP patterns. - -## 8. Component specifications - -### 8.1 Config (`internal/config`) - -- Path: `$XDG_CONFIG_HOME/cuttle/config.toml` (default `~/.config/cuttle/`). -- Selection precedence for the active context: `--context` flag > `CUTTLE_CONTEXT` - env > `default_context` in file > built-in `local`. -- Read with go-toml strict mode (typo protection). v1 is read-mostly; provide - `cuttle context ls` (list + show active). Defer write-back commands - (`context add/use`) to a later pass (they use `unstable/edit`). - -Schema: - -```toml -default_context = "cluster" - -[context.local] -backend = "local" # docker here; today's behavior - -[context.cluster] -backend = "k8s" -namespace = "browser" -release = "cuttle" -# kube_context omitted -> inherit kubectl's current context (which already -# carries the user's routing, e.g. a tailnet; cuttle stays ignorant of it) -node_selector = { "glim.sh/browser" = "true" } -tolerations = [{ key = "browser", operator = "Exists", effect = "NoSchedule" }] -resources = { requests = { memory = "1Gi" }, limits = { memory = "4Gi" } } -proxy = "http://user:pass@proxy.example:8080" # applied at browser startup - -[context.box] -backend = "ssh" -host = "misha@box.example" # inherits ~/.ssh/config, keys, jump hosts -proxy = "..." - -[context.tailnet] -backend = "direct" -cdp_url = "http://cuttle.ts.net:9222" -vnc_url = "http://cuttle.ts.net:6080" - -# Profiles (= cuttle seeds). Optional; omitted profiles behave as ephemeral. -[profile.default] -storage = "local" # local-canonical (default). "remote" = persist on the browser host. -[profile.linkedin] -storage = "local" -``` - -### 8.2 cloakbrowser port (`internal/fingerprint`) - STEALTH-CRITICAL - -Port these from `packages/cuttle-py/vendor/cloakbrowser/`: -- `browser.py` -> arg building: `build_args`, proxy URL normalization - (`_ensure_proxy_scheme`, `_assemble_proxy_url`, `_reconstruct_socks_url`, - `_normalize_socks_string_url`), `_resolve_webrtc_args` (replace - `--fingerprint-webrtc-ip=auto` with the resolved proxy exit IP). -- `geoip.py` -> `resolve_proxy_geo_with_ip` (tz/locale/exit-IP), the - `COUNTRY_LOCALE_MAP`, and mmdb download+cache (see 8.3). -- `config.py` -> the small config/env resolution + `CLOAKBROWSER_BINARY_PATH` - override (`download.py` collapses to "read the env path, error clearly"). - -MANDATORY parity requirement: the Go `build_args` output (the full Chrome argv, -including `--fingerprint-*`, `--proxy-server`, `--user-data-dir`, ordering) MUST -be byte-identical to the Python `build_args` for a matrix of inputs (seed x -proxy present/absent x timezone/locale set/unset x webrtc auto/none). Write a -table-driven Go test that asserts this against golden argv captured from the -Python oracle. A silent drift here = silent stealth loss (no error, just -detection). geoip resolution and proxy-URL normalization get the same -parity-test treatment. - -### 8.3 GeoIP mmdb download - -Mirror the Python: download `GeoLite2-City.mmdb` (~70 MB) on first use from the -P3TERX mirror (`https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb`), -cache under `$XDG_CACHE_HOME/cuttle/` (or the geoip cache dir), background -re-download after 30 days. Never block browser startup on the download; a -missing DB degrades to "no tz/locale" but still returns the exit IP (used for -WebRTC). Read via `geoip2-golang/v2`. - -### 8.4 `cuttle serve` (the multiplexer, `internal/serve`) - the daemon - -This is the in-container daemon, folded in as a `cuttle serve` subcommand (the -image entrypoint). Port `packages/cuttle-py/bin/cuttleserve`. It is a CDP -multiplexer: one Chrome process per fingerprint seed, all fronted on one port. - -Behaviors to port faithfully (the subtle, battle-tested parts): -- HTTP endpoints `/json/version` and `/json` that return CDP target info. - **Rewrite the `webSocketDebuggerUrl` host to the request's Host header** so it - is correct behind `kubectl port-forward` / `ssh -L` (this fixes the port-forward - wrinkle by construction - the Python fought a sibling loopback-bind bug). -- Connect-URL query parsing: `?fingerprint=&timezone=..&locale=..&proxy=..`. - Seed validation regex `^[A-Za-z0-9_-]{1,128}$`; reserved seed `__default__`. -- Per-seed Chrome pool: spawn one Chrome per seed with `user_data_dir = - {data_dir}/{seed}`, connection refcounting, and **idle reap only when - idle-timeout > 0** (local only; remote leaves it 0 = never reap). -- WebSocket Origin allow-list (verify it accepts `kubectl port-forward` origins, - which are typically `127.0.0.1`). -- Clean Chrome drain on SIGTERM (the CLI stops with a grace period > the internal - drain so a clean exit is recorded and no junk-tab crash-restore happens). -- Bind host: `0.0.0.0` in a container (detected), `127.0.0.1` otherwise. Env - override `CUTTLESERVE_HOST`. (The Python already handles the k8s/containerd - detection; port it.) -- The three cuttle-specific patches on top of upstream cloakserve (see - `packages/cuttle-py/docs/UPSTREAM.md`), port each deliberately: - 1. proxy-auth-over-CDP injection (answering the proxy auth challenge via CDP), - 2. service_worker `browserContextId` stamp, - 3. fork launch-parity flags. - -NEW in the Go serve: -- `--proxy` / `CUTTLESERVE_PROXY`: a server-level DEFAULT proxy. A connection - that omits `?proxy=` inherits it. This is what makes proxy a `cuttle up`/config - concern applied uniformly regardless of backend. geoip keys off the effective - proxy, so tz/locale follow it automatically. -- Ephemeral profile dir support (see 8.6): when a profile is local-canonical, - the seed's `user_data_dir` is an ephemeral scratch (emptyDir/tmpfs); nothing - sensitive persists on the remote. - -### 8.5 CLI + `reach()` + backends (`internal/cli`, `internal/backend`) - -Model the whole thing on one seam: a `Backend` obtains a reachable local CDP -endpoint. Every CDP/VNC-facing operation runs against `127.0.0.1:`. - -``` -type Backend interface { - State(ctx) (string, error) // running / stopped / absent - Start(ctx, opts) error - Stop(ctx, purge bool) error - Reach(ctx) (Endpoint, release func(), error) // yields local cdp/vnc ports -} -``` - -Implementations: - -| Backend | State | Start | Stop | Reach | -|---|---|---|---|---| -| local | `docker inspect` | `docker run` (port map, shm, keep-profile) | `docker stop`/`rm` | direct 127.0.0.1, no tunnel | -| k8s | `kubectl get pod` | `helm upgrade --install ops/helm/cuttle` | `helm upgrade --set replicaCount=0` (`--purge` = `helm uninstall` + delete PVC) | `kubectl port-forward svc/cuttle` -> ephemeral local port | -| ssh | `ssh host docker inspect` | `ssh host docker run` | `ssh host docker stop` | `ssh -L` (ControlMaster) -> ephemeral local port | -| direct | CDP probe of config URL | error (not cuttle-managed) | error | parse `cdp_url`/`vnc_url` from config, use as-is | - -Port today's `cuttle/cli.py` docker logic verbatim into the `local` backend so -current behavior does not regress (default context = `local` with no config -file present). - -Reachability rules: -- `status` / `login` open an EPHEMERAL forward for the command's duration - (stateless, matches cuttle's "container is the state" ethos). -- NEW `cuttle connect`: holds the forward open in the foreground, prints local - CDP/VNC URLs + the driver attach briefing, Ctrl-C to end. This is what an - agent/interactive session uses to avoid re-forwarding per call. -- Auto-pick a free local port for forwards so a k8s attach never collides with a - `local` container already on 9222. - -Presence checks (mirror today's `_docker()`): verify `docker`/`kubectl`/`helm`/ -`ssh` on PATH per backend, with clear errors. Thread `kube_context` onto every -kubectl/helm invocation when set. - -Preserve the existing "driver briefing" output (agent-facing: prints installed -CDP drivers - agent-browser / browser-use / playwright-cli - with attach lines -and self-doc commands). Port the `_DRIVERS` table. - -### 8.6 Profiles = seeds; local-canonical storage_state (`internal/profile`) - -Named profile == cuttle seed. `--profile ` threads through `login`, -`connect`, `mcp`, and every attach line as `?fingerprint=`. Profile data -lives locally at `$XDG_DATA_HOME/cuttle/profiles//storage_state.json`. - -The unit is a SESSION (attach -> detach), NOT a request - a browser holds -cookies in memory across many requests; there is no per-request injection seam. - -local-canonical flow (default): -1. Checkout (session start): read the local profile's storage_state (cookies + - per-origin localStorage/IndexedDB - Playwright storageState shape, small - JSON) and inject into the freshly spawned remote seed over CDP - (`Network.SetCookies` + `Runtime.Evaluate` per origin for localStorage). The - remote seed uses an ephemeral user_data_dir. -2. Session runs; nothing sensitive persists on the remote. -3. Checkin (session end + periodic checkpoint): extract the updated - storage_state via CDP (`Network.GetCookies`, read localStorage per origin) - and write it back locally. This is where snowballed cookies land. - -Rules & honest caveats to implement: -- Single-writer lock per profile (local lockfile): a profile checked out to one - remote session cannot be concurrently attached elsewhere (Chrome's own rule). -- Periodic checkpoint (every N minutes) during long sessions so a crash before - checkin does not lose the cookie delta. -- storage_state covers cookie/localStorage/IndexedDB auth (enough for - LinkedIn/X/GitHub). It does NOT capture full Chrome-profile fidelity - (extensions, SW caches, device-bound state). Provide a full-dir sync escape - hatch only if a site needs it (heavy; requires Chrome stopped for a consistent - snapshot). Not v1-critical. -- "Resides locally" is true AT REST. During an active session the live cookies - are necessarily on the remote (the browser must hold them to act as the user). - This is inherent to remote browsing; document it, do not pretend otherwise. - -remote-persistent opt-in (`storage = "remote"`): the seed's user_data_dir is a -durable path on the browser host; no checkout/checkin. Needed for autonomous / -always-on use where the user's machine is not present to inject state. For k8s -this means a PVC (see 8.8), which reintroduces RWO/Recreate/SingletonLock -handling ONLY for remote-persistent profiles. - -### 8.7 `cuttle mcp` (`internal/mcp`) - -- `cuttle mcp [driver] --profile `: ensure the driver is installed - (default browser-use: `uv tool install browser-use`) and write the MCP client - config (JSON, stdlib) pointing at the active context's CDP endpoint with the - profile seed, e.g. `BU_CDP_URL=http://127.0.0.1:9222?fingerprint=` (plus - the context proxy already applied server-side). -- Reuse the `_DRIVERS` table for install commands + attach templates. - -### 8.8 Dockerfile (`packages/cuttle-go/Dockerfile`) - Python-free - -Port `packages/cuttle-py/Dockerfile` but replace the Python runtime with a -static Go binary: -- Multi-stage: build the `cuttle` binary (`CGO_ENABLED=0 go build -trimpath - -ldflags="-s -w"`), copy into a slim base. -- Keep the browser-engine stages verbatim (SHA-pinned clark/clearcote tarball - downloads to `/opt/clark` and `/opt/clearcote`, `CLOAKBROWSER_BINARY_PATH`). -- Keep the headed stack (Xvfb/Xvnc, openbox, noVNC/websockify, the Windows font - pack) and the entrypoint that starts Xvnc + `cuttle serve`. -- linux/amd64 only (clark/clearcote ship linux-x64). Result: no Python, no uv, - no pip, no venv in the image. -- Note: `scripts/rename-fonts.py` is a build-time font step; either keep it as a - build-only Python invocation or reimplement with a Go/CLI font tool. Low - priority; keep the Python build step if simpler. - -### 8.9 Helm chart (`ops/helm/cuttle/`) - -Bundle-agnostic (the CLI shells `helm upgrade --install` against this path; -consider embedding via `embed.FS` so an installed binary carries the chart). - -Templates: -- Deployment: the cuttle image running `cuttle serve` (keep-profile only for - remote-persistent profiles), `replicas: 1`, `CUTTLESERVE_HOST=0.0.0.0`, the - context proxy passed via `CUTTLESERVE_PROXY`, `nodeSelector`, `tolerations`, - `resources`, and a HARD `nodeSelector: kubernetes.io/arch: amd64` (the image is - amd64-only; never let it schedule on arm). -- Volume: - - Default (local-canonical profiles): `emptyDir` for the profile scratch - - nothing persists, no RWO/Recreate/SingletonLock headache. - - remote-persistent profiles: a PVC (RWO) at the data dir, `strategy: Recreate` - (RWO cannot mount into a new pod while the old holds it), stale Chrome - `SingletonLock` cleared on boot, `helm.sh/resource-policy: keep`. -- Service: ClusterIP exposing CDP 9222 + VNC 6080 (reached via port-forward; no - ingress, nothing public). -- NetworkPolicy: deny ingress to the pod except from nothing (port-forward does - not traverse the Service network path). CDP has no auth; anyone who can reach - it drives your logged-in browser - lock it down. -- values.yaml: `image.tag` pinned to the CLI version, `nodeSelector`, - `tolerations`, `resources`, `dataDir`, `proxy`, `replicaCount`, - `profileStorage` (local|remote). - -## 9. Stealth fidelity: the safety net - -The rewrite's only real risk is silently degrading stealth. Mitigations, all -mandatory before deleting `cuttle-py`: -1. Parity tests (8.2): Go `build_args` / geoip / proxy-normalization output == - Python oracle output, byte-for-byte, across an input matrix. Golden files - captured from the Python. -2. STEALTH-VERIFICATION pass: bring up a Go-built seed and run the checklist in - `packages/cuttle-py/docs/STEALTH-VERIFICATION.md` (navigator.webdriver=false, - coherent Win32 + Chrome UA, real ANGLE/Direct3D11 WebGL renderer NOT - SwiftShader/llvmpipe, WebRTC only the proxy exit IP). Do NOT add - `--enable-unsafe-swiftshader` (it is a stealth regression, not a fix). -3. Live challenge clear: confirm a Go-driven seed clears an escalated anti-bot - challenge headed-on-Xvnc, same as Python. Remember: cold-clear is dominated by - the proxy exit-IP reputation, not the fingerprint - test on a clean exit. - -## 10. Build sequence (phases, on branch `feat/go-rewrite`) - -Keep `packages/cuttle-py/` working as the oracle until the very end. - -1. Restructure: `git mv` current tree into `packages/cuttle-py/`; create - `packages/cuttle-go/` scaffold with the go-dev toolchain; `ops/helm/` - placeholder. One commit, Python still builds. -2. `internal/fingerprint`: port build_args + proxy-normalize + geoip + webrtc. - Land the parity tests against the Python oracle FIRST - this is the - stealth-critical core, lock it before anything depends on it. -3. `internal/serve`: the CDP multiplexer + default-proxy + ws-host rewrite + - ephemeral-dir support + the three UPSTREAM patches. Smoke it against a real - clark browser locally over CDP. -4. `internal/config` + `internal/backend` (local first, proving no-regression, - then k8s, ssh, direct) + `internal/cli` (up/down/status/login/connect, - driver briefing). -5. `internal/profile`: storage_state checkout/checkin over CDP + locking + - checkpoints; wire local-canonical default; remote-persistent opt-in. -6. `internal/mcp`: install + JSON config write for the active context/profile. -7. Dockerfile (Python-free) + Helm chart (`ops/helm/cuttle/`, emptyDir default, - PVC for remote-persistent) + docs (port README/SKILL/AGENTS/CHANGELOG, - retire UPSTREAM.md + scripts/sync.sh). -8. Parity + STEALTH-VERIFICATION + live-challenge pass. Then remove - `packages/cuttle-py/` and promote the Go module to repo root - (`github.com/glim-sh/cuttle`). - -## 11. Testing strategy - -- Pure functions (fingerprint args, proxy normalize, geoip mapping, connect-URL - assembly, config resolution, backend command construction): table-driven unit - tests, `-race`. These carry the correctness weight and are fully testable - without infra. -- Parity tests: golden argv/geoip captured from the Python oracle (a small - script in cuttle-py dumps them for a fixed input matrix). -- Backend dispatch: mock the exec layer (inject a command runner) so - local/k8s/ssh/direct routing + flag construction are tested without docker/ - kubectl/ssh present. -- Integration (manual, documented checklist, not CI): real local clark browser - smoke; real k8s bring-up over port-forward; browser-use + agent-browser attach - through a forward (verifies the ws-host rewrite); STEALTH-VERIFICATION. -- CI: go-dev GitHub Actions (lint + test matrix + govulncheck). Integration - stays manual (needs a cluster + browser). - -## 12. Open decisions for the owner (resolve before/at build start) - -1. Config format: DECIDED - TOML (`go-toml/v2`). -2. CLI framework: cobra (rich UX, a dep) vs stdlib flag (minimal). Plan assumes - cobra. -3. `rename-fonts.py`: keep the tiny Python build-step in the Go image, or - reimplement in Go? (Plan: keep it; it is build-time only.) - -## 13. Source map (Python -> Go) - -| Python (packages/cuttle-py) | Go target (packages/cuttle-go) | -|---|---| -| `cuttle/cli.py` | `cmd/cuttle/main.go` + `internal/cli/*` + `internal/backend/local.go` | -| `cuttle/cdp.py` | `internal/cdp/*` (chromedp RemoteAllocator + coder/websocket) | -| `cuttle/view.py` (experimental CDP-screencast viewer) | `internal/view/*` (optional; port last or drop for v1) | -| `bin/cuttleserve` | `internal/serve/*` (the `cuttle serve` subcommand) | -| `vendor/cloakbrowser/browser.py` | `internal/fingerprint/args.go` + `proxy.go` | -| `vendor/cloakbrowser/geoip.py` | `internal/fingerprint/geoip.go` | -| `vendor/cloakbrowser/config.py` + `download.py` | `internal/fingerprint/binary.go` | -| `Dockerfile` | `packages/cuttle-go/Dockerfile` | -| `docs/UPSTREAM.md` + `scripts/sync.sh` | RETIRED (we own the code now) | -| `SKILL.md` / `AGENTS.md` / `README.md` | ported, updated for Go + backends | - -Key existing behaviors to preserve are cited by file:line in -`packages/cuttle-py/` - read `bin/cuttleserve` (bind/detect, seed pool, drain), -`vendor/cloakbrowser/browser.py` (build_args, webrtc), and `cuttle/cli.py` -(docker lifecycle, driver briefing) before porting each. -``` diff --git a/flake.lock b/flake.lock deleted file mode 100644 index fe90317..0000000 --- a/flake.lock +++ /dev/null @@ -1,27 +0,0 @@ -{ - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1783522502, - "narHash": "sha256-iffAls3iaNTyJC2faYcUXSI+Gp02cDjYl+MygxKl2GI=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "0bb7ec54c8483066ec9d7720e780a5caa71f8612", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 397d6e4..0000000 --- a/flake.nix +++ /dev/null @@ -1,62 +0,0 @@ -{ - description = "cuttle - host CLI for the stealth-Chromium CDP farm"; - - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - - outputs = - { self, nixpkgs }: - let - # The CLI is pure Python, so it builds from source on every platform - - # unlike the Docker image (the actual browser farm), which is amd64-only. - systems = [ - "aarch64-darwin" - "x86_64-darwin" - "x86_64-linux" - "aarch64-linux" - ]; - forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); - version = (builtins.fromTOML (builtins.readFile ./pyproject.toml)).project.version; - mkCuttle = - pkgs: - pkgs.python312.pkgs.buildPythonApplication { - pname = "cuttle-browser"; - inherit version; - pyproject = true; - src = self; - build-system = [ pkgs.python312.pkgs.setuptools ]; - # Mirrors [project.dependencies]; the container-only `server` group - # (httpx/geoip2/socksio) is deliberately absent - see pyproject.toml. - dependencies = with pkgs.python312.pkgs; [ - aiohttp - websockets - ]; - # The test harness needs a running cuttle container; nothing to run here. - doCheck = false; - pythonImportsCheck = [ "cuttle" ]; - meta = { - description = "Host CLI for the cuttle stealth-Chromium CDP farm"; - homepage = "https://github.com/glim-sh/cuttle"; - changelog = "https://github.com/glim-sh/cuttle/releases/tag/v${version}"; - license = nixpkgs.lib.licenses.mit; - mainProgram = "cuttle"; - }; - }; - in - { - overlays.default = final: _prev: { cuttle = mkCuttle final; }; - - packages = forAllSystems (pkgs: rec { - cuttle = mkCuttle pkgs; - default = cuttle; - }); - - apps = forAllSystems (pkgs: rec { - cuttle = { - type = "app"; - program = "${self.packages.${pkgs.system}.cuttle}/bin/cuttle"; - meta.description = "Run the cuttle host CLI"; - }; - default = cuttle; - }); - }; -} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cfb52a1 --- /dev/null +++ b/go.mod @@ -0,0 +1,27 @@ +module github.com/glim-sh/cuttle + +go 1.26 + +toolchain go1.26.5 + +require ( + github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f + github.com/chromedp/chromedp v0.16.0 + github.com/coder/websocket v1.8.15 + github.com/oschwald/geoip2-golang/v2 v2.2.0 + github.com/pelletier/go-toml/v2 v2.4.3 + github.com/spf13/cobra v1.10.2 + golang.org/x/net v0.57.0 +) + +require ( + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/oschwald/maxminddb-golang/v2 v2.4.1 // indirect + github.com/spf13/pflag v1.0.10 // indirect + golang.org/x/sys v0.47.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9b1bfb8 --- /dev/null +++ b/go.sum @@ -0,0 +1,50 @@ +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk= +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0= +github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk= +github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/oschwald/geoip2-golang/v2 v2.2.0 h1:gdkhpnHQMiH9ymOI+zSB0QKFGH+n4TntNt7vz+TxGPY= +github.com/oschwald/geoip2-golang/v2 v2.2.0/go.mod h1:xW4tCeQiNU1gqMD1x7zEH2CDNM3d796Ls50yxYDaX0U= +github.com/oschwald/maxminddb-golang/v2 v2.4.1 h1:OffzqSABE3Sw354GdBThqDsKfpA4GWBqOY2P91V8tjI= +github.com/oschwald/maxminddb-golang/v2 v2.4.1/go.mod h1:CZK8iQQMKfy6mKOifoyUmrj4vTHnMiGVaS7hDaZZxQ0= +github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= +github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/backend/backend.go b/internal/backend/backend.go new file mode 100644 index 0000000..0d7b7ca --- /dev/null +++ b/internal/backend/backend.go @@ -0,0 +1,147 @@ +// Package backend obtains a reachable local CDP/VNC endpoint for a browser that +// runs in one of four places: a local docker container, a Kubernetes Deployment, +// docker on an ssh host, or a pre-exposed direct URL. Every CDP/VNC-facing +// operation runs against the Endpoint a backend yields, so the rest of cuttle is +// transport-agnostic. +package backend + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "strings" + + "github.com/glim-sh/cuttle/internal/config" +) + +// Shared literals for the docker/kubectl/ssh command construction. +const ( + loopbackHost = "127.0.0.1" + dockerExe = "docker" + helmInstall = "--install" +) + +// State is a browser's lifecycle state as a backend sees it. +type State string + +const ( + StateRunning State = "running" + StateStopped State = "stopped" + StateAbsent State = "absent" +) + +// dockerStatusState maps `docker inspect -f {{.State.Status}}` output and its +// exit code to a State: a non-zero exit or empty status is an absent container, +// "running" is running, anything else is stopped. Shared by the local and ssh +// backends. +func dockerStatusState(status string, code int) State { + if code != 0 { + return StateAbsent + } + switch strings.TrimSpace(status) { + case "": + return StateAbsent + case string(StateRunning): + return StateRunning + default: + return StateStopped + } +} + +// Endpoint is a reachable CDP (and optional VNC) address. For tunneled backends +// the host is loopback and the ports are auto-picked local forwards; for direct +// it is the configured host/port as-is. +type Endpoint struct { + CDPHost string + CDPPort int + VNCHost string + VNCPort int // 0 = no VNC +} + +// StartOpts carries the per-invocation choices for Start. Not every field +// applies to every backend (e.g. IdleTimeout is local-only, Recreate is docker- +// only); a backend ignores what it does not use. +type StartOpts struct { + Image string + Recreate bool + KeepProfile *bool // nil = backend default (on) + NoVNC bool + Proxy string + IdleTimeout string // local only + Storage string // profile storage: "local" | "remote" +} + +// Backend manages one browser's lifecycle and reachability. +type Backend interface { + State(ctx context.Context) (State, error) + Start(ctx context.Context, opts StartOpts) error + Stop(ctx context.Context, purge bool) error + // Reach yields a local endpoint plus a release func that tears down any + // tunnel opened to reach it. release is always safe to call (no-op for + // direct/local). cdpPort/vncPort request specific local ports for a tunneled + // backend (k8s/ssh) so a held forward is deterministic and `cuttle mcp` can + // point a driver at it; 0 auto-picks a free port (used by the ephemeral + // status/login/up forwards, which never collide with a local container). + // local/direct ignore the requested ports and return their fixed endpoint. + Reach(ctx context.Context, cdpPort, vncPort int) (Endpoint, func(), error) +} + +var ( + errNotManaged = errors.New("not managed by cuttle") + errUnknownBackend = errors.New("unknown backend") + errNoTCPAddr = errors.New("listener address is not TCP") +) + +// New builds the backend for a resolved context. Ports are the host-side CDP/VNC +// ports for the local backend (and the remote container ports for ssh). +func New(name string, ctx config.Context, r Runner, cdpPort, vncPort int, image string) (Backend, error) { + switch ctx.Backend { + case config.BackendLocal, "": + return &Local{runner: r, name: name, cdpPort: cdpPort, vncPort: vncPort, image: image}, nil + case config.BackendK8s: + return newK8s(ctx, r), nil + case config.BackendSSH: + return &SSH{runner: r, host: ctx.Host, name: name, cdpPort: cdpPort, vncPort: vncPort, image: image, proxy: ctx.Proxy}, nil + case config.BackendDirect: + return newDirect(ctx) + default: + return nil, fmt.Errorf("%w %q for context %q", errUnknownBackend, ctx.Backend, name) + } +} + +// freePort picks a free local TCP port by binding :0 and releasing it. There is +// an inherent race between release and reuse, but it makes a forward collision +// with an existing local container on a fixed port (9222) vanishingly unlikely. +func freePort() (int, error) { + var lc net.ListenConfig + l, err := lc.Listen(context.Background(), "tcp", loopbackHost+":0") + if err != nil { + return 0, fmt.Errorf("picking free port: %w", err) + } + defer func() { _ = l.Close() }() + addr, ok := l.Addr().(*net.TCPAddr) + if !ok { + return 0, errNoTCPAddr + } + return addr.Port, nil +} + +// chooseLocalPort returns the requested local port, or a free one when preferred +// is 0. +func chooseLocalPort(preferred int) (int, error) { + if preferred > 0 { + return preferred, nil + } + return freePort() +} + +func requireExe(r Runner, exe, hint string) error { + if _, err := r.LookPath(exe); err != nil { + return fmt.Errorf("%s not found on PATH - %s", exe, hint) //nolint:err113 // operator-facing detail + } + return nil +} + +func portStr(p int) string { return strconv.Itoa(p) } diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go new file mode 100644 index 0000000..ad805a7 --- /dev/null +++ b/internal/backend/backend_test.go @@ -0,0 +1,572 @@ +package backend + +import ( + "context" + "slices" + "strings" + "sync" + "testing" + + "github.com/glim-sh/cuttle/internal/config" +) + +// mockRunner records every command and answers Output via a programmable hook. +type mockRunner struct { + mu sync.Mutex + calls [][]string + started [][]string + respond func(name string, args []string) Result + absent map[string]bool +} + +func (m *mockRunner) Output(_ context.Context, name string, args ...string) (Result, error) { + m.mu.Lock() + m.calls = append(m.calls, append([]string{name}, args...)) + m.mu.Unlock() + if m.respond != nil { + return m.respond(name, args), nil + } + return Result{}, nil +} + +func (m *mockRunner) Start(_ context.Context, name string, args ...string) (Process, error) { + m.mu.Lock() + m.started = append(m.started, append([]string{name}, args...)) + m.mu.Unlock() + return noopProcess{}, nil +} + +func (m *mockRunner) LookPath(name string) (string, error) { + if m.absent != nil && m.absent[name] { + return "", errFakeMissing + } + return "/usr/bin/" + name, nil +} + +type noopProcess struct{} + +func (noopProcess) Stop() error { return nil } + +var errFakeMissing = &missingError{} + +type missingError struct{} + +func (*missingError) Error() string { return "not found" } + +// lastCall returns the most recent recorded call whose first two tokens match +// the given verb path, e.g. lastCall("docker", "run"). +func (m *mockRunner) lastCall(prefix ...string) []string { + m.mu.Lock() + defer m.mu.Unlock() + for _, v := range slices.Backward(m.calls) { + if hasPrefixTokens(v, prefix) { + return v + } + } + return nil +} + +func hasPrefixTokens(call, prefix []string) bool { + if len(call) < len(prefix) { + return false + } + // prefix matches the command name plus the first meaningful verb, allowing + // intervening global flags (e.g. kubectl --context X get ...). + if call[0] != prefix[0] { + return false + } + rest := call[1:] + for _, want := range prefix[1:] { + if !slices.Contains(rest, want) { + return false + } + } + return true +} + +func assertArgv(t *testing.T, got, want []string) { + t.Helper() + if !slices.Equal(got, want) { + t.Fatalf("argv mismatch:\n got: %v\nwant: %v", got, want) + } +} + +// --------------------------------------------------------------------------- +// local +// --------------------------------------------------------------------------- + +func TestLocalStartFreshRun(t *testing.T) { + tests := []struct { + name string + opts StartOpts + wantTail []string + }{ + { + name: "default keep-profile and vnc", + opts: StartOpts{Image: "img:1"}, + wantTail: []string{ + "docker", "run", "-d", "--name", "cuttle", + "-p", "127.0.0.1:9222:9222", "--shm-size=2g", + "-p", "127.0.0.1:6080:6080", "-e", "CUTTLE_VNC=1", + "img:1", "cuttleserve", "--keep-profile", + }, + }, + { + name: "no-vnc and proxy, keep-profile off", + opts: StartOpts{Image: "img:1", NoVNC: true, Proxy: "http://p:1", KeepProfile: new(bool)}, + wantTail: []string{ + "docker", "run", "-d", "--name", "cuttle", + "-p", "127.0.0.1:9222:9222", "--shm-size=2g", + "-e", "CUTTLESERVE_PROXY=http://p:1", + "img:1", "cuttleserve", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &mockRunner{respond: dockerAbsent} + l := &Local{runner: r, name: "cuttle", cdpPort: 9222, vncPort: 6080, image: "fallback:0"} + if err := l.Start(context.Background(), tt.opts); err != nil { + t.Fatalf("Start: %v", err) + } + assertArgv(t, r.lastCall("docker", "run"), tt.wantTail) + }) + } +} + +func TestLocalStartRestartsExited(t *testing.T) { + r := &mockRunner{respond: func(_ string, args []string) Result { + if slices.Contains(args, "inspect") { + return Result{Stdout: "exited\n"} + } + return Result{} + }} + l := &Local{runner: r, name: "cuttle", cdpPort: 9222, vncPort: 6080} + if err := l.Start(context.Background(), StartOpts{}); err != nil { + t.Fatalf("Start: %v", err) + } + assertArgv(t, r.lastCall("docker", "start"), []string{"docker", "start", "cuttle"}) + if r.lastCall("docker", "run") != nil { + t.Fatal("exited container should restart, not run") + } +} + +func TestLocalStopArgv(t *testing.T) { + tests := []struct { + name string + purge bool + want [][]string + }{ + {"graceful", false, [][]string{{"docker", "stop", "-t", "15", "cuttle"}}}, + {"purge", true, [][]string{{"docker", "stop", "-t", "15", "cuttle"}, {"docker", "rm", "-f", "cuttle"}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &mockRunner{respond: func(_ string, args []string) Result { + if slices.Contains(args, "inspect") { + return Result{Stdout: "running\n"} + } + return Result{} + }} + l := &Local{runner: r, name: "cuttle"} + if err := l.Stop(context.Background(), tt.purge); err != nil { + t.Fatalf("Stop: %v", err) + } + assertArgv(t, r.lastCall("docker", "stop"), tt.want[0]) + if tt.purge { + assertArgv(t, r.lastCall("docker", "rm"), tt.want[1]) + } + }) + } +} + +func TestLocalStateMapping(t *testing.T) { + tests := []struct { + status string + code int + want State + }{ + {"", 1, StateAbsent}, + {"running", 0, StateRunning}, + {"exited", 0, StateStopped}, + {"created", 0, StateStopped}, + } + for _, tt := range tests { + t.Run(string(tt.want)+"_"+tt.status, func(t *testing.T) { + r := &mockRunner{respond: func(string, []string) Result { + return Result{Stdout: tt.status, Code: tt.code} + }} + l := &Local{runner: r, name: "cuttle"} + got, err := l.State(context.Background()) + if err != nil { + t.Fatalf("State: %v", err) + } + if got != tt.want { + t.Fatalf("got %q want %q", got, tt.want) + } + }) + } +} + +func TestLocalMissingDocker(t *testing.T) { + r := &mockRunner{absent: map[string]bool{"docker": true}} + l := &Local{runner: r, name: "cuttle"} + if _, err := l.State(context.Background()); err == nil || !strings.Contains(err.Error(), "docker") { + t.Fatalf("want docker-not-found error, got %v", err) + } +} + +// --------------------------------------------------------------------------- +// k8s +// --------------------------------------------------------------------------- + +func k8sContext() config.Context { + return config.Context{ + Backend: config.BackendK8s, + Namespace: "browser", + Release: "cuttle", + KubeContext: "kind", + NodeSelector: map[string]string{"glim.sh/browser": "true"}, + } +} + +func TestK8sStartArgv(t *testing.T) { + r := &mockRunner{} + k := newK8s(k8sContext(), r) + if err := k.Start(context.Background(), StartOpts{Proxy: "http://u:p@proxy.example:8080"}); err != nil { + t.Fatalf("Start: %v", err) + } + want := []string{ + "helm", "--kube-context", "kind", "--namespace", "browser", + "upgrade", "--install", "cuttle", "ops/helm/cuttle", "--create-namespace", + "--set", "replicaCount=1", + "--set-string", "proxy=http://u:p@proxy.example:8080", + "--set-string", "profileStorage=local", + "--set-string", `nodeSelector.glim\.sh/browser=true`, + } + assertArgv(t, r.lastCall("helm", "upgrade"), want) +} + +func TestK8sStopArgv(t *testing.T) { + t.Run("scale down", func(t *testing.T) { + r := &mockRunner{} + k := newK8s(k8sContext(), r) + if err := k.Stop(context.Background(), false); err != nil { + t.Fatalf("Stop: %v", err) + } + assertArgv(t, r.lastCall("helm", "upgrade"), []string{ + "helm", "--kube-context", "kind", "--namespace", "browser", + "upgrade", "--install", "cuttle", "ops/helm/cuttle", "--reuse-values", "--set", "replicaCount=0", + }) + }) + t.Run("purge uninstalls and deletes pvc", func(t *testing.T) { + r := &mockRunner{} + k := newK8s(k8sContext(), r) + if err := k.Stop(context.Background(), true); err != nil { + t.Fatalf("Stop: %v", err) + } + assertArgv(t, r.lastCall("helm", "uninstall"), []string{ + "helm", "--kube-context", "kind", "--namespace", "browser", "uninstall", "cuttle", + }) + assertArgv(t, r.lastCall("kubectl", "delete"), []string{ + "kubectl", "--context", "kind", "-n", "browser", + "delete", "pvc", "-l", "app.kubernetes.io/instance=cuttle", + }) + }) +} + +func TestK8sStatePhase(t *testing.T) { + tests := []struct { + phases string + code int + want State + }{ + {"", 1, StateAbsent}, + {"Running", 0, StateRunning}, + {"Pending", 0, StateStopped}, + } + for _, tt := range tests { + t.Run(string(tt.want), func(t *testing.T) { + r := &mockRunner{respond: func(string, []string) Result { + return Result{Stdout: tt.phases, Code: tt.code} + }} + k := newK8s(k8sContext(), r) + got, err := k.State(context.Background()) + if err != nil { + t.Fatalf("State: %v", err) + } + if got != tt.want { + t.Fatalf("got %q want %q", got, tt.want) + } + }) + } + // verify the jsonpath query argv + r := &mockRunner{} + k := newK8s(k8sContext(), r) + _, _ = k.State(context.Background()) + assertArgv(t, r.lastCall("kubectl", "get"), []string{ + "kubectl", "--context", "kind", "-n", "browser", + "get", "pod", "-l", "app.kubernetes.io/instance=cuttle", "-o", "jsonpath={.items[*].status.phase}", + }) +} + +func TestK8sReachPortForward(t *testing.T) { + r := &mockRunner{} + k := newK8s(k8sContext(), r) + ep, release, err := k.Reach(context.Background(), 0, 0) + if err != nil { + t.Fatalf("Reach: %v", err) + } + defer release() + if ep.CDPHost != "127.0.0.1" || ep.CDPPort == 0 || ep.VNCPort == 0 || ep.CDPPort == ep.VNCPort { + t.Fatalf("bad endpoint: %+v", ep) + } + if len(r.started) != 1 { + t.Fatalf("want one port-forward, got %d", len(r.started)) + } + pf := r.started[0] + if !slices.Equal(pf[:6], []string{"kubectl", "--context", "kind", "-n", "browser", "port-forward"}) { + t.Fatalf("port-forward prefix: %v", pf) + } + if !slices.Contains(pf, "svc/cuttle") { + t.Fatalf("missing svc target: %v", pf) + } + if !strings.HasSuffix(pf[len(pf)-2], ":9222") || !strings.HasSuffix(pf[len(pf)-1], ":6080") { + t.Fatalf("port mappings: %v", pf[len(pf)-2:]) + } +} + +func TestK8sReachPinnedPorts(t *testing.T) { + r := &mockRunner{} + k := newK8s(k8sContext(), r) + ep, release, err := k.Reach(context.Background(), 9333, 6081) + if err != nil { + t.Fatalf("Reach: %v", err) + } + defer release() + if ep.CDPPort != 9333 || ep.VNCPort != 6081 { + t.Fatalf("pinned ports not honored: %+v", ep) + } + pf := r.started[0] + if pf[len(pf)-2] != "9333:9222" || pf[len(pf)-1] != "6081:6080" { + t.Fatalf("port-forward should use pinned local ports: %v", pf[len(pf)-2:]) + } +} + +// --------------------------------------------------------------------------- +// ssh +// --------------------------------------------------------------------------- + +func sshBackend(r Runner) *SSH { + return &SSH{runner: r, host: "user@box.example", name: "cuttle", cdpPort: 9222, vncPort: 6080, image: "img:1"} +} + +func TestSSHStateArgv(t *testing.T) { + r := &mockRunner{respond: func(string, []string) Result { return Result{Stdout: "running"} }} + s := sshBackend(r) + got, err := s.State(context.Background()) + if err != nil { + t.Fatalf("State: %v", err) + } + if got != StateRunning { + t.Fatalf("state: %q", got) + } + call := r.lastCall("ssh") + cp := s.controlPath() + want := []string{ + "ssh", "-o", "ControlMaster=auto", "-o", "ControlPath=" + cp, "user@box.example", + "docker", "inspect", "-f", "{{.State.Status}}", "cuttle", + } + assertArgv(t, call, want) +} + +func TestSSHStartArgv(t *testing.T) { + r := &mockRunner{} + s := sshBackend(r) + if err := s.Start(context.Background(), StartOpts{NoVNC: true, KeepProfile: new(bool)}); err != nil { + t.Fatalf("Start: %v", err) + } + cp := s.controlPath() + want := []string{ + "ssh", "-o", "ControlMaster=auto", "-o", "ControlPath=" + cp, "user@box.example", + "docker", "run", "-d", "--name", "cuttle", + "-p", "127.0.0.1:9222:9222", "--shm-size=2g", + "img:1", "cuttleserve", + } + assertArgv(t, r.lastCall("ssh"), want) +} + +func TestSSHReachTunnelArgv(t *testing.T) { + r := &mockRunner{} + s := sshBackend(r) + ep, release, err := s.Reach(context.Background(), 0, 0) + if err != nil { + t.Fatalf("Reach: %v", err) + } + defer release() + if ep.CDPPort == 0 || ep.VNCPort == 0 || ep.CDPPort == ep.VNCPort { + t.Fatalf("bad endpoint: %+v", ep) + } + tun := r.started[0] + if tun[0] != "ssh" || tun[len(tun)-1] != "user@box.example" { + t.Fatalf("tunnel argv: %v", tun) + } + if !slices.Contains(tun, "-N") { + t.Fatalf("tunnel missing -N: %v", tun) + } + // two -L forwards ending in the remote container ports + var forwards []string + for i, a := range tun { + if a == "-L" && i+1 < len(tun) { + forwards = append(forwards, tun[i+1]) + } + } + if len(forwards) != 2 || !strings.HasSuffix(forwards[0], ":127.0.0.1:9222") || !strings.HasSuffix(forwards[1], ":127.0.0.1:6080") { + t.Fatalf("forwards: %v", forwards) + } +} + +// --------------------------------------------------------------------------- +// direct +// --------------------------------------------------------------------------- + +func TestDirectStartStopError(t *testing.T) { + d, err := newDirect(config.Context{Backend: config.BackendDirect, CDPURL: "http://cuttle.example:9222"}) + if err != nil { + t.Fatalf("newDirect: %v", err) + } + if err := d.Start(context.Background(), StartOpts{}); err == nil { + t.Fatal("Start should error for direct") + } + if err := d.Stop(context.Background(), false); err == nil { + t.Fatal("Stop should error for direct") + } +} + +func TestDirectReachUsesConfigURLs(t *testing.T) { + d, err := newDirect(config.Context{ + Backend: config.BackendDirect, + CDPURL: "http://cuttle.example:9222", + VNCURL: "https://cuttle.example:6080", + }) + if err != nil { + t.Fatalf("newDirect: %v", err) + } + ep, release, err := d.Reach(context.Background(), 0, 0) + if err != nil { + t.Fatalf("Reach: %v", err) + } + defer release() + if ep.CDPHost != "cuttle.example" || ep.CDPPort != 9222 || ep.VNCPort != 6080 { + t.Fatalf("endpoint: %+v", ep) + } +} + +func TestDirectRequiresCDPURL(t *testing.T) { + if _, err := newDirect(config.Context{Backend: config.BackendDirect}); err == nil { + t.Fatal("expected error when cdp_url missing") + } +} + +func TestDirectStateProbe(t *testing.T) { + d, err := newDirect(config.Context{Backend: config.BackendDirect, CDPURL: "http://cuttle.example:9222"}) + if err != nil { + t.Fatalf("newDirect: %v", err) + } + d.probe = func(context.Context, string) bool { return true } + if s, _ := d.State(context.Background()); s != StateRunning { + t.Fatalf("want running, got %q", s) + } + d.probe = func(context.Context, string) bool { return false } + if s, _ := d.State(context.Background()); s != StateAbsent { + t.Fatalf("want absent, got %q", s) + } +} + +// --------------------------------------------------------------------------- +// shared +// --------------------------------------------------------------------------- + +func TestFreePortDistinctAndUsable(t *testing.T) { + seen := map[int]bool{} + for range 5 { + p, err := freePort() + if err != nil { + t.Fatalf("freePort: %v", err) + } + if p <= 0 || p > 65535 { + t.Fatalf("bad port %d", p) + } + seen[p] = true + } + if len(seen) < 2 { + t.Fatalf("expected varied ports, got %v", seen) + } +} + +func TestEscapeHelm(t *testing.T) { + segTests := []struct{ in, want string }{ + {"replicaCount", "replicaCount"}, + {"glim.sh/browser", `glim\.sh/browser`}, + {"a,b", `a\,b`}, + } + for _, tt := range segTests { + if got := escapeHelmSegment(tt.in); got != tt.want { + t.Fatalf("escapeHelmSegment(%q)=%q want %q", tt.in, got, tt.want) + } + } + // Values keep dots/slashes/colons; only commas are structural. + if got := escapeHelmValue("http://u:p@proxy.example:8080"); got != "http://u:p@proxy.example:8080" { + t.Fatalf("escapeHelmValue dropped/added escaping: %q", got) + } + if got := escapeHelmValue("a,b"); got != `a\,b` { + t.Fatalf("escapeHelmValue(comma)=%q", got) + } +} + +func TestNewDispatch(t *testing.T) { + r := &mockRunner{} + tests := []struct { + backend string + ctx config.Context + want string + }{ + {"local", config.Context{Backend: config.BackendLocal}, "*backend.Local"}, + {"k8s", config.Context{Backend: config.BackendK8s}, "*backend.K8s"}, + {"ssh", config.Context{Backend: config.BackendSSH, Host: "h"}, "*backend.SSH"}, + {"direct", config.Context{Backend: config.BackendDirect, CDPURL: "http://x:9222"}, "*backend.Direct"}, + } + for _, tt := range tests { + t.Run(tt.backend, func(t *testing.T) { + b, err := New(tt.backend, tt.ctx, r, 9222, 6080, "img") + if err != nil { + t.Fatalf("New: %v", err) + } + if got := typeName(b); got != tt.want { + t.Fatalf("got %s want %s", got, tt.want) + } + }) + } +} + +func dockerAbsent(_ string, args []string) Result { + if slices.Contains(args, "inspect") { + return Result{Code: 1} + } + return Result{} +} + +func typeName(v any) string { + switch v.(type) { + case *Local: + return "*backend.Local" + case *K8s: + return "*backend.K8s" + case *SSH: + return "*backend.SSH" + case *Direct: + return "*backend.Direct" + default: + return "unknown" + } +} diff --git a/internal/backend/direct.go b/internal/backend/direct.go new file mode 100644 index 0000000..430271b --- /dev/null +++ b/internal/backend/direct.go @@ -0,0 +1,108 @@ +package backend + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/glim-sh/cuttle/internal/config" +) + +// Direct targets a pre-exposed CDP/VNC endpoint from config as-is. It is the +// escape hatch for any browser cuttle does not itself manage (e.g. reached over +// a tailnet), so Start/Stop are errors. +type Direct struct { + cdpHost string + cdpPort int + vncHost string + vncPort int + cdpURL string + // probe reports whether the CDP endpoint answers; injectable for tests. + probe func(ctx context.Context, cdpURL string) bool +} + +func newDirect(ctx config.Context) (*Direct, error) { + if ctx.CDPURL == "" { + return nil, errors.New("direct context requires cdp_url") //nolint:err113 + } + cdpHost, cdpPort, err := splitHostPort(ctx.CDPURL) + if err != nil { + return nil, fmt.Errorf("invalid cdp_url %q: %w", ctx.CDPURL, err) + } + d := &Direct{cdpHost: cdpHost, cdpPort: cdpPort, cdpURL: ctx.CDPURL, probe: probeCDP} + if ctx.VNCURL != "" { + vncHost, vncPort, err := splitHostPort(ctx.VNCURL) + if err != nil { + return nil, fmt.Errorf("invalid vnc_url %q: %w", ctx.VNCURL, err) + } + d.vncHost, d.vncPort = vncHost, vncPort + } + return d, nil +} + +func (d *Direct) State(ctx context.Context) (State, error) { + if d.probe(ctx, d.cdpURL) { + return StateRunning, nil + } + return StateAbsent, nil +} + +func (d *Direct) Start(context.Context, StartOpts) error { + return fmt.Errorf("direct context is a pre-exposed endpoint, %w - start the browser yourself", errNotManaged) +} + +func (d *Direct) Stop(context.Context, bool) error { + return fmt.Errorf("direct context is a pre-exposed endpoint, %w - stop the browser yourself", errNotManaged) +} + +// Reach uses the configured URLs as-is; there is no tunnel to release. +func (d *Direct) Reach(context.Context, int, int) (Endpoint, func(), error) { + return Endpoint{ + CDPHost: d.cdpHost, CDPPort: d.cdpPort, + VNCHost: d.vncHost, VNCPort: d.vncPort, + }, func() {}, nil +} + +func splitHostPort(rawURL string) (string, int, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", 0, err //nolint:wrapcheck + } + host := u.Hostname() + if host == "" { + return "", 0, errors.New("missing host") //nolint:err113 + } + port := u.Port() + if port == "" { + if u.Scheme == "https" { + return host, 443, nil + } + return host, 80, nil + } + p, err := strconv.Atoi(port) + if err != nil { + return "", 0, fmt.Errorf("invalid port %q", port) //nolint:err113 + } + return host, p, nil +} + +func probeCDP(ctx context.Context, cdpURL string) bool { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, cdpURL+"/json/version", nil) + if err != nil { + return false + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16)) + return resp.StatusCode == http.StatusOK +} diff --git a/internal/backend/k8s.go b/internal/backend/k8s.go new file mode 100644 index 0000000..a4c72c2 --- /dev/null +++ b/internal/backend/k8s.go @@ -0,0 +1,218 @@ +package backend + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/glim-sh/cuttle/internal/config" +) + +// chartPath is the Helm chart the k8s backend installs. Relative to the repo +// root; a packaged binary would embed it. +const chartPath = "ops/helm/cuttle" + +const instanceSelector = "app.kubernetes.io/instance=" + +// defaultRelease is the Helm release name when a k8s context omits `release`. +const defaultRelease = "cuttle" + +// K8s runs the browser as a Helm-managed Deployment in a cluster, reached via +// kubectl port-forward. It shells out to kubectl/helm and inherits the user's +// kube context (and thus their routing) with zero cuttle-specific setup. +type K8s struct { + runner Runner + namespace string + release string + ctx config.Context +} + +func newK8s(ctx config.Context, r Runner) *K8s { + release := ctx.Release + if release == "" { + release = defaultRelease + } + namespace := ctx.Namespace + if namespace == "" { + namespace = "default" + } + return &K8s{runner: r, namespace: namespace, release: release, ctx: ctx} +} + +func (k *K8s) check() error { + if err := requireExe(k.runner, "kubectl", "install kubectl and configure a cluster context first."); err != nil { + return err + } + return requireExe(k.runner, "helm", "install Helm first.") +} + +// kubectlArgs threads the kube context and namespace onto every kubectl call. +func (k *K8s) kubectlArgs(args ...string) []string { + out := []string{} + if k.ctx.KubeContext != "" { + out = append(out, "--context", k.ctx.KubeContext) + } + out = append(out, "-n", k.namespace) + return append(out, args...) +} + +// helmArgs threads the kube context and namespace onto every helm call. +func (k *K8s) helmArgs(args ...string) []string { + out := []string{} + if k.ctx.KubeContext != "" { + out = append(out, "--kube-context", k.ctx.KubeContext) + } + out = append(out, "--namespace", k.namespace) + return append(out, args...) +} + +func (k *K8s) State(ctx context.Context) (State, error) { + if err := k.check(); err != nil { + return "", err + } + res, err := k.runner.Output(ctx, "kubectl", k.kubectlArgs( + "get", "pod", "-l", instanceSelector+k.release, "-o", "jsonpath={.items[*].status.phase}", + )...) + if err != nil { + return "", err + } + phases := strings.TrimSpace(res.Stdout) + if res.Code != 0 || phases == "" { + return StateAbsent, nil + } + if strings.Contains(phases, "Running") { + return StateRunning, nil + } + return StateStopped, nil +} + +func (k *K8s) Start(ctx context.Context, opts StartOpts) error { + if err := k.check(); err != nil { + return err + } + setArgs := k.installSets(opts) + args := k.helmArgs(append([]string{"upgrade", helmInstall, k.release, chartPath, "--create-namespace"}, setArgs...)...) + return runOK(ctx, k.runner, "helm upgrade", "helm", args...) +} + +// installSets builds the --set flags for the chart from the context config and +// start options. Keys are dotted paths (a dot separates path segments), so only +// dynamic segments (map keys) are escaped; values only need comma-escaping. +// Map/list entries are emitted in a stable (sorted / indexed) order so the argv +// is deterministic. +func (k *K8s) installSets(opts StartOpts) []string { + var sets []string + set := func(key, v string) { sets = append(sets, "--set", key+"="+escapeHelmValue(v)) } + setStr := func(key, v string) { sets = append(sets, "--set-string", key+"="+escapeHelmValue(v)) } + + set("replicaCount", "1") + if opts.Image != "" { + if _, tag, ok := strings.Cut(opts.Image, ":"); ok { + setStr("image.tag", tag) + } + } + if opts.Proxy != "" { + setStr("proxy", opts.Proxy) + } + storage := opts.Storage + if storage == "" { + storage = config.StorageLocal + } + setStr("profileStorage", storage) + + for _, key := range sortedKeys(k.ctx.NodeSelector) { + setStr("nodeSelector."+escapeHelmSegment(key), k.ctx.NodeSelector[key]) + } + for i, tol := range k.ctx.Tolerations { + prefix := fmt.Sprintf("tolerations[%d].", i) + if tol.Key != "" { + setStr(prefix+"key", tol.Key) + } + if tol.Operator != "" { + setStr(prefix+"operator", tol.Operator) + } + if tol.Value != "" { + setStr(prefix+"value", tol.Value) + } + if tol.Effect != "" { + setStr(prefix+"effect", tol.Effect) + } + } + if k.ctx.Resources != nil { + for _, key := range sortedKeys(k.ctx.Resources.Requests) { + setStr("resources.requests."+escapeHelmSegment(key), k.ctx.Resources.Requests[key]) + } + for _, key := range sortedKeys(k.ctx.Resources.Limits) { + setStr("resources.limits."+escapeHelmSegment(key), k.ctx.Resources.Limits[key]) + } + } + return sets +} + +func (k *K8s) Stop(ctx context.Context, purge bool) error { + if err := k.check(); err != nil { + return err + } + if !purge { + args := k.helmArgs("upgrade", helmInstall, k.release, chartPath, "--reuse-values", "--set", "replicaCount=0") + return runOK(ctx, k.runner, "helm scale-down", "helm", args...) + } + + if err := runOK(ctx, k.runner, "helm uninstall", "helm", k.helmArgs("uninstall", k.release)...); err != nil { + return err + } + return runOK(ctx, k.runner, "kubectl delete pvc", "kubectl", k.kubectlArgs("delete", "pvc", "-l", instanceSelector+k.release)...) +} + +// Reach opens a kubectl port-forward. cdpPort/vncPort pin the local ports (so a +// held `cuttle connect` forward is deterministic and mcp can target it); 0 +// auto-picks free ports for the ephemeral status/login forwards, which then +// never collide with a local container already on 9222. +func (k *K8s) Reach(ctx context.Context, cdpPort, vncPort int) (Endpoint, func(), error) { + if err := k.check(); err != nil { + return Endpoint{}, nil, err + } + cdpLocal, err := chooseLocalPort(cdpPort) + if err != nil { + return Endpoint{}, nil, err + } + vncLocal, err := chooseLocalPort(vncPort) + if err != nil { + return Endpoint{}, nil, err + } + args := k.kubectlArgs( + "port-forward", "svc/"+k.release, + portStr(cdpLocal)+":"+containerCDPPort, + portStr(vncLocal)+":"+containerVNCPort, + ) + proc, err := k.runner.Start(ctx, "kubectl", args...) + if err != nil { + return Endpoint{}, nil, fmt.Errorf("starting port-forward: %w", err) + } + ep := Endpoint{CDPHost: loopbackHost, CDPPort: cdpLocal, VNCHost: loopbackHost, VNCPort: vncLocal} + return ep, func() { _ = proc.Stop() }, nil +} + +// escapeHelmSegment escapes a single --set key path segment: dots (which would +// otherwise split the path) and commas, so a map key like "glim.sh/browser" +// stays one segment. +func escapeHelmSegment(s string) string { + return strings.NewReplacer(".", `\.`, ",", `\,`).Replace(s) +} + +// escapeHelmValue escapes only what separates --set values: commas. Dots, +// slashes, and colons are literal in values, so a credentialed proxy URL passes +// through intact. +func escapeHelmValue(s string) string { + return strings.ReplaceAll(s, ",", `\,`) +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} diff --git a/internal/backend/local.go b/internal/backend/local.go new file mode 100644 index 0000000..6432d8f --- /dev/null +++ b/internal/backend/local.go @@ -0,0 +1,201 @@ +package backend + +import ( + "context" + "strings" +) + +const ( + containerCDPPort = "9222" + containerVNCPort = "6080" + shmSize = "--shm-size=2g" + stopGrace = "15" // > cuttleserve's 5s Chrome drain, so the clean exit completes + serveCommand = "cuttleserve" + dockerRunSub = "run" + dockerNameFlag = "--name" +) + +// Local runs the browser in a docker container on this host. It is a faithful +// port of the Python cuttle CLI's docker lifecycle, so existing behavior does +// not regress when no config file is present. +type Local struct { + runner Runner + name string + cdpPort int + vncPort int + image string // resolved default image, used when StartOpts.Image is empty +} + +func (l *Local) check() error { + return requireExe(l.runner, dockerExe, "install Docker (or OrbStack) first.") +} + +// dockerStatus returns the container's raw docker state ("running", "exited", +// "created", ...) or "" if the container does not exist. +func (l *Local) dockerStatus(ctx context.Context) (string, error) { + res, err := l.runner.Output(ctx, dockerExe, "inspect", "-f", "{{.State.Status}}", l.name) + if err != nil { + return "", err + } + if res.Code != 0 { + return "", nil + } + return strings.TrimSpace(res.Stdout), nil +} + +func (l *Local) State(ctx context.Context) (State, error) { + if err := l.check(); err != nil { + return "", err + } + status, err := l.dockerStatus(ctx) + if err != nil { + return "", err + } + // dockerStatus already folds a non-zero exit into an empty status. + return dockerStatusState(status, 0), nil +} + +func (l *Local) rm(ctx context.Context) { + _, _ = l.runner.Output(ctx, dockerExe, "rm", "-f", l.name) +} + +// Start ensures the container is up, idempotently. A stopped container is +// restarted (profile preserved); a zombie (a run that died before a clean exit) +// is removed and re-run; --recreate forces a fresh container. +func (l *Local) Start(ctx context.Context, opts StartOpts) error { + if err := l.check(); err != nil { + return err + } + status, err := l.dockerStatus(ctx) + if err != nil { + return err + } + + if opts.Recreate && status != "" { + l.rm(ctx) + status = "" + } + // A container that never ran cleanly ("created" from a run that died at + // network setup, or "dead") has no live host port binding to restart into. + // Only "exited" is safe to restart (a clean `cuttle down`). + if status != "" && status != string(StateRunning) && status != "exited" { + l.rm(ctx) + status = "" + } + + switch { + case status == string(StateRunning): + return nil + case status != "": // exited -> restart, keeping the profile + return runOK(ctx, l.runner, "docker start", dockerExe, "start", l.name) + } + + image := opts.Image + if image == "" { + image = l.image + } + args := dockerRunArgs(l.name, l.cdpPort, l.vncPort, opts, image) + if err := runOK(ctx, l.runner, "docker run", dockerExe, args...); err != nil { + // A run that fails at network setup leaves a half-created container + // behind; remove it so the next `up` does not mistake it for restartable. + l.rm(ctx) + return err + } + return nil +} + +// dockerRunArgs builds the `docker run ...` argv (without the leading "docker") +// shared by the local and ssh backends. The container always binds host +// 127.0.0.1: locally that is this machine, over ssh it is the remote host that +// ssh -L then tunnels to. +func dockerRunArgs(name string, cdpPort, vncPort int, opts StartOpts, image string) []string { + args := []string{ + dockerRunSub, "-d", + dockerNameFlag, name, + "-p", loopbackHost + ":" + portStr(cdpPort) + ":" + containerCDPPort, + shmSize, + } + if !opts.NoVNC { + args = append(args, "-p", loopbackHost+":"+portStr(vncPort)+":"+containerVNCPort, "-e", "CUTTLE_VNC=1") + } + if opts.Proxy != "" { + args = append(args, "-e", "CUTTLESERVE_PROXY="+opts.Proxy) + } + // cuttleserve defaults to port 9222 and auto-binds 0.0.0.0 in a container, so + // pass neither; it only accepts the `=` form and has no --host flag. + args = append(args, image, serveCommand) + if opts.IdleTimeout != "" { + args = append(args, "--idle-timeout="+opts.IdleTimeout) + } + if opts.KeepProfile == nil || *opts.KeepProfile { + args = append(args, "--keep-profile") + } + return args +} + +func (l *Local) Stop(ctx context.Context, purge bool) error { + if err := l.check(); err != nil { + return err + } + status, err := l.dockerStatus(ctx) + if err != nil { + return err + } + if status == "" { + return nil + } + if status == string(StateRunning) { + if err := runOK(ctx, l.runner, "docker stop", dockerExe, "stop", "-t", stopGrace, l.name); err != nil { + return err + } + } + if purge { + if err := runOK(ctx, l.runner, "docker rm", dockerExe, "rm", "-f", l.name); err != nil { + return err + } + } + return nil +} + +// Image reports the image an existing container was created with, or "". +func (l *Local) Image(ctx context.Context) string { + res, err := l.runner.Output(ctx, dockerExe, "inspect", "-f", "{{.Config.Image}}", l.name) + if err != nil || res.Code != 0 { + return "" + } + return strings.TrimSpace(res.Stdout) +} + +// Diagnostics returns human-readable triage lines for an unhealthy container: +// the real host<-container port bindings and a log tail, so triage never needs a +// raw docker command. It is used by `status` via an optional interface. +func (l *Local) Diagnostics(ctx context.Context) []string { + var lines []string + if res, err := l.runner.Output(ctx, dockerExe, "port", l.name); err == nil && res.Code == 0 { + if ports := strings.TrimSpace(res.Stdout); ports != "" { + lines = append(lines, "actual port bindings (start `up` with THESE ports, do not --recreate):") + for line := range strings.SplitSeq(ports, "\n") { + lines = append(lines, " "+line) + } + } + } + if res, err := l.runner.Output(ctx, dockerExe, "logs", "--tail", "20", l.name); err == nil { + out := strings.TrimSpace(res.Stdout + res.Stderr) + if out != "" { + lines = append(lines, "last 20 log lines:") + for line := range strings.SplitSeq(out, "\n") { + lines = append(lines, " "+line) + } + } + } + return lines +} + +// Reach for local is a direct loopback endpoint on the host-mapped ports; no +// tunnel, so release is a no-op. +func (l *Local) Reach(_ context.Context, _, _ int) (Endpoint, func(), error) { + return Endpoint{ + CDPHost: loopbackHost, CDPPort: l.cdpPort, + VNCHost: loopbackHost, VNCPort: l.vncPort, + }, func() {}, nil +} diff --git a/internal/backend/runner.go b/internal/backend/runner.go new file mode 100644 index 0000000..dc180e4 --- /dev/null +++ b/internal/backend/runner.go @@ -0,0 +1,97 @@ +package backend + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strings" +) + +// runOK runs a command through r and returns nil only if it both executed and +// exited zero; a non-zero exit becomes an error carrying label and the trimmed +// stderr. It collapses the run-and-check-exit shape every backend mutation uses. +func runOK(ctx context.Context, r Runner, label, name string, args ...string) error { + res, err := r.Output(ctx, name, args...) + if err != nil { + return err + } + if res.Code != 0 { + return fmt.Errorf("%s failed:\n%s", label, strings.TrimSpace(res.Stderr)) //nolint:err113 + } + return nil +} + +// Runner is the exec seam every backend goes through, so command construction is +// unit-testable without docker/kubectl/ssh installed. +type Runner interface { + // Output runs a command to completion and captures its output. A non-zero + // exit is reported in Result.Code with a nil error (mirroring the Python + // check=False); a nil error means only that the command ran. + Output(ctx context.Context, name string, args ...string) (Result, error) + // Start launches a long-running command (a tunnel) and returns a handle to + // stop it. + Start(ctx context.Context, name string, args ...string) (Process, error) + // LookPath reports the resolved path of an executable, or an error if absent. + LookPath(name string) (string, error) +} + +// Result is a finished command's captured output and exit code. +type Result struct { + Stdout string + Stderr string + Code int +} + +// Process is a running command that can be stopped. +type Process interface { + Stop() error +} + +// ExecRunner is the production Runner backed by os/exec. +type ExecRunner struct{} + +func (ExecRunner) Output(ctx context.Context, name string, args ...string) (Result, error) { + cmd := exec.CommandContext(ctx, name, args...) + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + res := Result{Stdout: stdout.String(), Stderr: stderr.String()} + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + res.Code = exitErr.ExitCode() + return res, nil + } + if err != nil { + return res, err //nolint:wrapcheck // the exec error is already descriptive + } + return res, nil +} + +func (ExecRunner) Start(ctx context.Context, name string, args ...string) (Process, error) { + cmd := exec.CommandContext(ctx, name, args...) + if err := cmd.Start(); err != nil { + return nil, err //nolint:wrapcheck + } + return &execProcess{cmd: cmd}, nil +} + +func (ExecRunner) LookPath(name string) (string, error) { + p, err := exec.LookPath(name) + if err != nil { + return "", err //nolint:wrapcheck + } + return p, nil +} + +type execProcess struct{ cmd *exec.Cmd } + +func (e *execProcess) Stop() error { + if e.cmd.Process == nil { + return nil + } + _ = e.cmd.Process.Kill() + _ = e.cmd.Wait() + return nil +} diff --git a/internal/backend/ssh.go b/internal/backend/ssh.go new file mode 100644 index 0000000..341120c --- /dev/null +++ b/internal/backend/ssh.go @@ -0,0 +1,122 @@ +package backend + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +const sshControlMaster = "ControlMaster=auto" + +// SSH runs the browser in docker on a remote host reached over ssh, tunneled to +// this machine with ssh -L. It inherits ~/.ssh/config (keys, jump hosts, and any +// routing the user provides), so cuttle needs no ssh setup of its own. +type SSH struct { + runner Runner + host string + name string + cdpPort int // remote host-published CDP port + vncPort int // remote host-published VNC port + image string + proxy string +} + +func (s *SSH) check() error { + return requireExe(s.runner, "ssh", "install an OpenSSH client and configure the host in ~/.ssh/config.") +} + +// controlPath is a deterministic ControlMaster socket per host, so State/Stop +// reuse the multiplexed connection the forwarding session established. +func (s *SSH) controlPath() string { + safe := strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + return r + } + return '_' + }, s.host) + return filepath.Join(os.TempDir(), "cuttle-ssh-"+safe+".sock") +} + +// remoteArgs runs a command on the ssh host, reusing the ControlMaster socket. +func (s *SSH) remoteArgs(cmd ...string) []string { + out := make([]string, 0, 5+len(cmd)) + out = append(out, "-o", sshControlMaster, "-o", "ControlPath="+s.controlPath(), s.host) + return append(out, cmd...) +} + +func (s *SSH) State(ctx context.Context) (State, error) { + if err := s.check(); err != nil { + return "", err + } + res, err := s.runner.Output(ctx, "ssh", s.remoteArgs(dockerExe, "inspect", "-f", "{{.State.Status}}", s.name)...) + if err != nil { + return "", err + } + return dockerStatusState(res.Stdout, res.Code), nil +} + +func (s *SSH) Start(ctx context.Context, opts StartOpts) error { + if err := s.check(); err != nil { + return err + } + if opts.Proxy == "" { + opts.Proxy = s.proxy + } + image := opts.Image + if image == "" { + image = s.image + } + run := dockerRunArgs(s.name, s.cdpPort, s.vncPort, opts, image) + return runOK(ctx, s.runner, "remote docker run", "ssh", s.remoteArgs(append([]string{dockerExe}, run...)...)...) +} + +func (s *SSH) Stop(ctx context.Context, purge bool) error { + if err := s.check(); err != nil { + return err + } + if err := runOK(ctx, s.runner, "remote docker stop", "ssh", s.remoteArgs(dockerExe, "stop", "-t", stopGrace, s.name)...); err != nil { + return err + } + if purge { + if err := runOK(ctx, s.runner, "remote docker rm", "ssh", s.remoteArgs(dockerExe, "rm", "-f", s.name)...); err != nil { + return err + } + } + return nil +} + +// Reach opens an ssh -L tunnel from local ports to the remote container's +// published ports, establishing the ControlMaster the other calls reuse. +// cdpPort/vncPort pin the local ports (so a held `cuttle connect` forward is +// deterministic and mcp can target it); 0 auto-picks free ports for the +// ephemeral status/login forwards. +func (s *SSH) Reach(ctx context.Context, cdpPort, vncPort int) (Endpoint, func(), error) { + if err := s.check(); err != nil { + return Endpoint{}, nil, err + } + cdpLocal, err := chooseLocalPort(cdpPort) + if err != nil { + return Endpoint{}, nil, err + } + vncLocal, err := chooseLocalPort(vncPort) + if err != nil { + return Endpoint{}, nil, err + } + args := []string{ + "-N", + "-o", sshControlMaster, + "-o", "ControlPath=" + s.controlPath(), + "-o", "ControlPersist=60", + "-L", portStr(cdpLocal) + ":127.0.0.1:" + portStr(s.cdpPort), + "-L", portStr(vncLocal) + ":127.0.0.1:" + portStr(s.vncPort), + s.host, + } + proc, err := s.runner.Start(ctx, "ssh", args...) + if err != nil { + return Endpoint{}, nil, fmt.Errorf("starting ssh tunnel: %w", err) + } + ep := Endpoint{CDPHost: loopbackHost, CDPPort: cdpLocal, VNCHost: loopbackHost, VNCPort: vncLocal} + return ep, func() { _ = proc.Stop() }, nil +} diff --git a/internal/cdp/cdp.go b/internal/cdp/cdp.go new file mode 100644 index 0000000..16fc4a8 --- /dev/null +++ b/internal/cdp/cdp.go @@ -0,0 +1,230 @@ +package cdp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/chromedp/cdproto/runtime" + "github.com/chromedp/cdproto/storage" + "github.com/chromedp/chromedp" +) + +// lsReadExpr returns the whole localStorage as a plain object so ReturnByValue +// serializes it to a {key:value} JSON map. +const lsReadExpr = "Object.assign({}, window.localStorage)" + +// The write expression is a marker-wrapped IIFE fed the items as a JSON literal; +// the marker/suffix let a test harness recover the payload deterministically. +const ( + lsWritePrefix = "/*cuttle-ls-write*/(function(d){for(var k in d){try{window.localStorage.setItem(k,d[k])}catch(e){}}})(" + lsWriteSuffix = ")" +) + +var ( + errEval = errors.New("localStorage evaluation failed") + errNoWSURL = errors.New("multiplexer did not return a webSocketDebuggerUrl") + errBadResponse = errors.New("bad /json/version response") +) + +func lsWriteExpr(items map[string]string) string { + b, _ := json.Marshal(items) + return lsWritePrefix + string(b) + lsWriteSuffix +} + +// getAllCookies and setCookies operate on browser-global cookies through the ctx +// executor, so the same code path is exercised by the real chromedp connection +// and by a fake CDP endpoint in tests. Storage.getCookies returns every cookie +// in the browser context, unlike Network.getCookies which is scoped to the +// current tab's URLs (empty on the scratch tab we connect on). +func getAllCookies(ctx context.Context) ([]*network.Cookie, error) { + cs, err := storage.GetCookies().Do(ctx) + if err != nil { + return nil, fmt.Errorf("Storage.getCookies: %w", err) + } + return cs, nil +} + +func setCookies(ctx context.Context, params []*network.CookieParam) error { + if len(params) == 0 { + return nil + } + if err := network.SetCookies(params).Do(ctx); err != nil { + return fmt.Errorf("Network.setCookies: %w", err) + } + return nil +} + +func readLocalStorage(ctx context.Context) (map[string]string, error) { + p := runtime.Evaluate(lsReadExpr) + p.ReturnByValue = true + res, exc, err := p.Do(ctx) + if err != nil { + return nil, fmt.Errorf("Runtime.evaluate(read): %w", err) + } + if exc != nil { + return nil, fmt.Errorf("%w: %s", errEval, exc.Text) + } + m := map[string]string{} + if res != nil && len(res.Value) > 0 { + if err := json.Unmarshal([]byte(res.Value), &m); err != nil { + return nil, fmt.Errorf("decoding localStorage: %w", err) + } + } + return m, nil +} + +func writeLocalStorage(ctx context.Context, items map[string]string) error { + if len(items) == 0 { + return nil + } + p := runtime.Evaluate(lsWriteExpr(items)) + _, exc, err := p.Do(ctx) + if err != nil { + return fmt.Errorf("Runtime.evaluate(write): %w", err) + } + if exc != nil { + return fmt.Errorf("%w: %s", errEval, exc.Text) + } + return nil +} + +// Extract connects to the seed's browser and reads its storage state. Cookies +// are browser-global; localStorage is origin-scoped, so each origin is visited +// on a scratch tab (localStorage is only readable from a document of that +// origin). An origin that fails to load contributes no localStorage rather than +// failing the whole extract; its origin string is returned in failed so the +// caller can tell a transient load failure apart from a genuinely-empty origin +// and preserve the last-known localStorage for the former. +func Extract(ctx context.Context, cdpBase, seed string, origins []string) (*StorageState, []string, error) { + taskCtx, cancel, err := connect(ctx, cdpBase, seed) + if err != nil { + return nil, nil, err + } + defer cancel() + + st := &StorageState{Cookies: []Cookie{}, Origins: []Origin{}} + if err := chromedp.Run(taskCtx, chromedp.ActionFunc(func(ctx context.Context) error { + cs, err := getAllCookies(ctx) + if err != nil { + return err + } + st.Cookies = fromCDPCookies(cs) + return nil + })); err != nil { + return nil, nil, err //nolint:wrapcheck // getAllCookies already wraps + } + + var failed []string + for _, origin := range origins { + var items map[string]string + read := chromedp.ActionFunc(func(ctx context.Context) error { + m, err := readLocalStorage(ctx) + items = m + return err + }) + if err := chromedp.Run(taskCtx, chromedp.Navigate(origin), read); err != nil { + failed = append(failed, origin) + continue + } + if len(items) > 0 { + st.Origins = append(st.Origins, Origin{Origin: origin, LocalStorage: mapToItems(items)}) + } + } + return st, failed, nil +} + +// Inject writes the storage state into the seed's fresh browser: cookies first +// (browser-global), then per-origin localStorage on a scratch tab navigated to +// each origin. +func Inject(ctx context.Context, cdpBase, seed string, st *StorageState) error { + taskCtx, cancel, err := connect(ctx, cdpBase, seed) + if err != nil { + return err + } + defer cancel() + + if err := chromedp.Run(taskCtx, chromedp.ActionFunc(func(ctx context.Context) error { + return setCookies(ctx, toCookieParams(st.Cookies)) + })); err != nil { + return err //nolint:wrapcheck // setCookies already wraps + } + + for _, o := range st.Origins { + items := itemsToMap(o.LocalStorage) + if len(items) == 0 { + continue + } + write := chromedp.ActionFunc(func(ctx context.Context) error { + return writeLocalStorage(ctx, items) + }) + if err := chromedp.Run(taskCtx, chromedp.Navigate(o.Origin), write); err != nil { + return fmt.Errorf("seeding localStorage for %s: %w", o.Origin, err) + } + } + return nil +} + +// connect resolves the seed's browser WebSocket URL through the multiplexer and +// opens a chromedp context bound to a fresh scratch tab. NoModifyURL keeps the +// resolved ?fingerprint routing intact, and the remote allocator guarantees +// chromedp attaches to the running browser instead of launching one. +func connect(ctx context.Context, cdpBase, seed string) (context.Context, context.CancelFunc, error) { + wsURL, err := browserWSURL(ctx, cdpBase, seed) + if err != nil { + return nil, nil, err + } + allocCtx, cancelAlloc := chromedp.NewRemoteAllocator(ctx, wsURL, chromedp.NoModifyURL) + taskCtx, cancelTask := chromedp.NewContext(allocCtx) + cancel := func() { + cancelTask() + cancelAlloc() + } + return taskCtx, cancel, nil +} + +// browserWSURL asks the multiplexer for the seed's browser CDP endpoint. The +// multiplexer rewrites webSocketDebuggerUrl to its own host, so the returned URL +// is correct behind a port-forward / ssh -L. +func browserWSURL(ctx context.Context, cdpBase, seed string) (string, error) { + base, err := url.Parse(cdpBase) + if err != nil { + return "", fmt.Errorf("parsing CDP base %q: %w", cdpBase, err) + } + base.Path = "/json/version" + if seed != "" { + base.RawQuery = "fingerprint=" + url.QueryEscape(seed) + } + + reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, base.String(), nil) + if err != nil { + return "", fmt.Errorf("building /json/version request: %w", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("reaching CDP: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", fmt.Errorf("reading /json/version: %w", err) + } + var v struct { + WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` + } + if err := json.Unmarshal(body, &v); err != nil { + return "", fmt.Errorf("%w: %w", errBadResponse, err) + } + if v.WebSocketDebuggerURL == "" { + return "", errNoWSURL + } + return v.WebSocketDebuggerURL, nil +} diff --git a/internal/cdp/cdp_test.go b/internal/cdp/cdp_test.go new file mode 100644 index 0000000..b096c0a --- /dev/null +++ b/internal/cdp/cdp_test.go @@ -0,0 +1,280 @@ +package cdp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/coder/websocket" +) + +var errRawCDP = errors.New("cdp error") + +func TestStorageStateRoundTrip(t *testing.T) { + t.Parallel() + srv, jar := fakeCDPServer(t) + exec := dialExec(t, srv.wsURL) + ctx := cdp.WithExecutor(t.Context(), exec) + + want := &StorageState{ + Cookies: []Cookie{ + {Name: "sid", Value: "abc", Domain: "example.com", Path: "/", Secure: true, HTTPOnly: true, SameSite: "Lax", Expires: 1893456000}, + {Name: "sess", Value: "z", Domain: "example.com", Path: "/", Expires: -1}, + }, + } + ls := map[string]string{"token": "t1", "theme": "dark"} + + if err := setCookies(ctx, toCookieParams(want.Cookies)); err != nil { + t.Fatalf("setCookies: %v", err) + } + if err := writeLocalStorage(ctx, ls); err != nil { + t.Fatalf("writeLocalStorage: %v", err) + } + + gotCookies, err := getAllCookies(ctx) + if err != nil { + t.Fatalf("getAllCookies: %v", err) + } + round := fromCDPCookies(gotCookies) + if len(round) != len(want.Cookies) { + t.Fatalf("cookie count: got %d want %d", len(round), len(want.Cookies)) + } + byName := map[string]Cookie{} + for _, c := range round { + byName[c.Name] = c + } + for _, w := range want.Cookies { + g, ok := byName[w.Name] + if !ok { + t.Fatalf("missing cookie %q", w.Name) + } + if g.Value != w.Value || g.Domain != w.Domain || g.Secure != w.Secure || g.HTTPOnly != w.HTTPOnly { + t.Fatalf("cookie %q mismatch: got %+v want %+v", w.Name, g, w) + } + } + + gotLS, err := readLocalStorage(ctx) + if err != nil { + t.Fatalf("readLocalStorage: %v", err) + } + if len(gotLS) != len(ls) { + t.Fatalf("localStorage: got %v want %v", gotLS, ls) + } + for k, v := range ls { + if gotLS[k] != v { + t.Fatalf("localStorage[%q]=%q want %q", k, gotLS[k], v) + } + } + _ = jar +} + +func TestBrowserWSURL(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/json/version" || r.URL.Query().Get("fingerprint") != "linkedin" { + http.Error(w, "unexpected", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{ + "webSocketDebuggerUrl": "ws://" + r.Host + "/fingerprint/linkedin/devtools/browser/abc", + }) + })) + defer srv.Close() + + got, err := browserWSURL(t.Context(), srv.URL, "linkedin") + if err != nil { + t.Fatalf("browserWSURL: %v", err) + } + if !strings.HasSuffix(got, "/fingerprint/linkedin/devtools/browser/abc") { + t.Fatalf("ws url: %q", got) + } +} + +func TestBrowserWSURLMissing(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"Browser": "Chrome/148"}) + })) + defer srv.Close() + if _, err := browserWSURL(t.Context(), srv.URL, "x"); !errors.Is(err, errNoWSURL) { + t.Fatalf("want errNoWSURL, got %v", err) + } +} + +func TestLSWriteExprRecoverable(t *testing.T) { + t.Parallel() + expr := lsWriteExpr(map[string]string{"a": "1"}) + payload := strings.TrimSuffix(strings.TrimPrefix(expr, lsWritePrefix), lsWriteSuffix) + var m map[string]string + if err := json.Unmarshal([]byte(payload), &m); err != nil || m["a"] != "1" { + t.Fatalf("payload=%q m=%v err=%v", payload, m, err) + } +} + +// --------------------------------------------------------------------------- +// fake CDP endpoint +// --------------------------------------------------------------------------- + +type fakeServer struct { + wsURL string +} + +type cookieJar struct { + mu sync.Mutex + cookies []map[string]any + ls map[string]string +} + +// fakeCDPServer speaks just enough CDP over a WebSocket to round-trip cookies and +// localStorage: it stores whatever setCookies/localStorage-write frames set and +// replays them on getAllCookies / localStorage-read. +func fakeCDPServer(t *testing.T) (*fakeServer, *cookieJar) { + t.Helper() + jar := &cookieJar{ls: map[string]string{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{InsecureSkipVerify: true}) + if err != nil { + return + } + defer func() { _ = conn.Close(websocket.StatusNormalClosure, "") }() + for { + _, data, err := conn.Read(r.Context()) + if err != nil { + return + } + reply := jar.handle(data) + if err := conn.Write(r.Context(), websocket.MessageText, reply); err != nil { + return + } + } + })) + t.Cleanup(srv.Close) + ws := "ws" + strings.TrimPrefix(srv.URL, "http") + return &fakeServer{wsURL: ws}, jar +} + +func (j *cookieJar) handle(data []byte) []byte { + var msg struct { + ID int64 `json:"id"` + Method string `json:"method"` + Params struct { + Cookies []map[string]any `json:"cookies"` + Expression string `json:"expression"` + } `json:"params"` + } + if err := json.Unmarshal(data, &msg); err != nil { + return []byte(`{"id":0,"result":{}}`) + } + j.mu.Lock() + defer j.mu.Unlock() + + result := map[string]any{} + switch msg.Method { + case "Network.setCookies": + j.cookies = append(j.cookies, msg.Params.Cookies...) + case "Storage.getCookies": + result["cookies"] = j.cookies + case "Runtime.evaluate": + result["result"] = j.evaluate(msg.Params.Expression) + } + out, _ := json.Marshal(map[string]any{"id": msg.ID, "result": result}) + return out +} + +func (j *cookieJar) evaluate(expr string) map[string]any { + switch { + case strings.HasPrefix(expr, lsWritePrefix): + payload := strings.TrimSuffix(strings.TrimPrefix(expr, lsWritePrefix), lsWriteSuffix) + var m map[string]string + if json.Unmarshal([]byte(payload), &m) == nil { + maps.Copy(j.ls, m) + } + return map[string]any{} + case expr == lsReadExpr: + return map[string]any{"value": j.ls} + default: + return map[string]any{} + } +} + +// --------------------------------------------------------------------------- +// raw CDP executor over one WebSocket (test transport for the low-level fns) +// --------------------------------------------------------------------------- + +type rawExecutor struct { + conn *websocket.Conn + mu sync.Mutex + id int64 +} + +func dialExec(t *testing.T, wsURL string) *rawExecutor { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + conn, resp, err := websocket.Dial(ctx, wsURL, nil) + if err != nil { + t.Fatalf("dial fake CDP: %v", err) + } + if resp != nil && resp.Body != nil { + _ = resp.Body.Close() + } + t.Cleanup(func() { _ = conn.Close(websocket.StatusNormalClosure, "") }) + return &rawExecutor{conn: conn} +} + +func (e *rawExecutor) Execute(ctx context.Context, method string, params, res any) error { + e.mu.Lock() + defer e.mu.Unlock() + e.id++ + p := json.RawMessage("{}") + if params != nil { + b, err := json.Marshal(params) + if err != nil { + return err + } + p = b + } + env, err := json.Marshal(struct { + ID int64 `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` + }{e.id, method, p}) + if err != nil { + return err + } + if err := e.conn.Write(ctx, websocket.MessageText, env); err != nil { + return err + } + for { + _, data, err := e.conn.Read(ctx) + if err != nil { + return err + } + var reply struct { + ID int64 `json:"id"` + Result json.RawMessage `json:"result"` + Error *struct { + Message string `json:"message"` + } `json:"error"` + } + if json.Unmarshal(data, &reply) != nil || reply.ID != e.id { + continue + } + if reply.Error != nil { + return fmt.Errorf("%w: %s", errRawCDP, reply.Error.Message) + } + if res != nil && len(reply.Result) > 0 { + return json.Unmarshal(reply.Result, res) + } + return nil + } +} diff --git a/internal/cdp/storage.go b/internal/cdp/storage.go new file mode 100644 index 0000000..208339d --- /dev/null +++ b/internal/cdp/storage.go @@ -0,0 +1,123 @@ +// Package cdp drives an already-running stealth browser over the Chrome DevTools +// Protocol to extract and inject browser storage state. It connects through the +// serve multiplexer's ?fingerprint= routing via chromedp's RemoteAllocator +// and never launches Chrome itself. +package cdp + +import ( + "slices" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/cdproto/network" +) + +// StorageState is the subset of the Playwright storageState JSON shape cuttle +// checks out and back in: global cookies plus per-origin localStorage. It is +// small (auth state, not full Chrome-profile fidelity) so it round-trips over +// CDP between the local canonical copy and an ephemeral remote seed. +type StorageState struct { + Cookies []Cookie `json:"cookies"` + Origins []Origin `json:"origins"` +} + +// Cookie mirrors a Playwright storageState cookie. Expires is seconds since the +// epoch, or -1 for a session cookie. +type Cookie struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Expires float64 `json:"expires"` + HTTPOnly bool `json:"httpOnly"` + Secure bool `json:"secure"` + SameSite string `json:"sameSite,omitempty"` +} + +// Origin is one origin's localStorage in the Playwright storageState shape. +type Origin struct { + Origin string `json:"origin"` + LocalStorage []LocalStorageItem `json:"localStorage"` +} + +// LocalStorageItem is a single localStorage key/value pair. +type LocalStorageItem struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// OriginURLs returns the origin strings the state carries, so a checkin knows +// which origins to re-read localStorage from. +func (s *StorageState) OriginURLs() []string { + out := make([]string, 0, len(s.Origins)) + for _, o := range s.Origins { + out = append(out, o.Origin) + } + return out +} + +func fromCDPCookies(cs []*network.Cookie) []Cookie { + out := make([]Cookie, 0, len(cs)) + for _, c := range cs { + if c == nil { + continue + } + out = append(out, Cookie{ + Name: c.Name, + Value: c.Value, + Domain: c.Domain, + Path: c.Path, + Expires: c.Expires, + HTTPOnly: c.HTTPOnly, + Secure: c.Secure, + SameSite: string(c.SameSite), + }) + } + return out +} + +func toCookieParams(cs []Cookie) []*network.CookieParam { + out := make([]*network.CookieParam, 0, len(cs)) + for _, c := range cs { + p := &network.CookieParam{ + Name: c.Name, + Value: c.Value, + Domain: c.Domain, + Path: c.Path, + Secure: c.Secure, + HTTPOnly: c.HTTPOnly, + } + if c.SameSite != "" { + p.SameSite = network.CookieSameSite(c.SameSite) + } + if c.Expires > 0 { + sec := int64(c.Expires) + nsec := int64((c.Expires - float64(sec)) * float64(time.Second)) + ts := cdp.TimeSinceEpoch(time.Unix(sec, nsec)) + p.Expires = &ts + } + out = append(out, p) + } + return out +} + +func itemsToMap(items []LocalStorageItem) map[string]string { + m := make(map[string]string, len(items)) + for _, it := range items { + m[it.Name] = it.Value + } + return m +} + +func mapToItems(m map[string]string) []LocalStorageItem { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + out := make([]LocalStorageItem, 0, len(m)) + for _, k := range keys { + out = append(out, LocalStorageItem{Name: k, Value: m[k]}) + } + return out +} diff --git a/internal/cli/SKILL.md b/internal/cli/SKILL.md new file mode 100644 index 0000000..d835506 --- /dev/null +++ b/internal/cli/SKILL.md @@ -0,0 +1,297 @@ +--- +name: cuttle +description: Run and drive cuttle - a local stealth-Chromium browser farm with persistent logins, anti-detect fingerprints, and a human-handoff viewer for captchas and Cloudflare. Use whenever the user says to use the browser, or asks to automate, scrape, test, or sign into a website, or names agent-browser, browser-use (bu, bu-cli), or playwright-cli. `cuttle up` prints the live briefing with installed drivers, exact CDP attach commands, and each driver's own docs command. Attach to cuttle's warm session - never launch a fresh browser or new profile. +metadata: + version: "0.3.0" # x-release-please-version + image: "ghcr.io/glim-sh/cuttle" +allowed-tools: Bash(cuttle:*) Bash(just:*) Bash(docker:*) Bash(curl:*) Bash(agent-browser:*) Bash(browser-use:*) Bash(playwright-cli:*) +--- + +# cuttle: local stealth-browser CDP farm + +[cuttle](https://github.com/glim-sh/cuttle) runs a patched CDP multiplexer +(`cuttle serve`) that spawns one stealth Chrome per fingerprint seed - each with +its own coherent identity (fingerprint, proxy, geoip, locale, timezone) - behind +a single CDP endpoint. The engine is a free stealth-Chromium fork (clark MIT, +default; clearcote BSD-3, fallback); no proprietary binary. Point any CDP client +at it - agent-browser, browser-use, Playwright, `chromium.connectOverCDP`. + +The `cuttle` CLI wraps a Docker container; it does not automate pages itself - +cuttle is the farm, not the scraper. Two ways to use it: + +- **Daily driver / login handoff** - one persistent browser you watch and log + into via VNC, driven over CDP. Use the CLI: `cuttle up`. Start below. +- **Multi-seed farm** - many isolated identities behind one endpoint, no VNC. + Run the container and pick a seed with `?fingerprint=`. See [Multi-seed farm](#multi-seed-farm). + +## Setup + +Requires Docker (or OrbStack). The CLI is a single static Go binary named +**`cuttle`**: + +```bash +brew install tenequm/tap/cuttle # homebrew cask (macOS/Linux) +go install github.com/glim-sh/cuttle/cmd/cuttle@latest # from source (needs Go 1.26+) +``` + +The container image is `ghcr.io/glim-sh/cuttle`; `cuttle up` pulls it on first +run. If you only want the raw farm without the CLI, run the image directly (see +[Multi-seed farm](#multi-seed-farm)). + +Then, from any directory: + +```bash +cuttle up # start the container with the VNC viewer on; pulls the image if needed +``` + +`up` (and `status`) print **the briefing**: CDP + viewer URLs, cuttle's +version, which driver CLIs are installed - with the exact attach command and +the self-doc command for each - plus routing rules and install advisories for +missing drivers. The briefing is the live source of truth; follow it over any +cached knowledge, including this guide. + +`up` is idempotent and profile-preserving: a stopped container is **restarted** +(logins persist), not recreated. Default ports: CDP 9222, VNC 6080. + +### Contexts and backends + +By default `cuttle` runs the browser in a **local** Docker container. It can also +run it remotely and reach it over a tunnel, selected by context (`--context`, or +`CUTTLE_CONTEXT`, or the config `default_context`): **local** (Docker), **k8s** +(a Deployment reached via `kubectl port-forward`), **ssh** (a container on a +remote host over `ssh -L`), and **direct** (an already-running CDP endpoint). +Every CDP/VNC op still targets a local `127.0.0.1:` - the backend owns the +tunnel. `cuttle context ls` lists contexts and marks the active one. + +**Local-canonical profiles.** Auth state (`--profile `) lives on your +machine and is injected into the session over CDP at login, then checked back in +- so logins are portable across backends and survive a container being +discarded. Omitted profiles behave as ephemeral. + +### Picking ports (important) + +Every subcommand takes `--cdp-port` and `--vnc-port`. Use them when the defaults +are taken: + +```bash +cuttle up --cdp-port 9444 --vnc-port 6099 +``` + +- The CLI is **stateless** - pass the *same* ports (and `--name`) to + `status`/`login`/`down` as you gave `up`, or they target the default 9222/6080. +- **Port-shadow gotcha:** `docker run` errors on a docker-vs-docker port clash, + but **not** when a *native* process (e.g. a local CDP shim on 9333) already owns + the host port. `cuttle up` then prints a mapping that is silently dead - your + client hits the other process, not cuttle. Verify with + `lsof -nP -iTCP: -sTCP:LISTEN` (want the owner to be OrbStack/Docker), or + just check `curl http://127.0.0.1:/json/version` reports the engine you + expect. Pick a genuinely free port when unsure. + +## Drive it (drivers + routing) + +cuttle serves standard CDP on `http://127.0.0.1:`. Drive it with a +driver CLI - the briefing lists the ones actually installed, the attach command +for each, and the command that prints that driver's own usage guide. + +- **Attach, never spawn.** Connect to cuttle's running browser and its default + context. Never launch your own Chromium and never create a new profile or + context - logins live in this one session and persist across restarts. cuttle + is a CDP endpoint, not a Chrome binary: never point a driver's + `--executable-path` at it, and do not pass a local `--profile` next to `--cdp` + (drivers reject that - the profile lives in cuttle's container). +- **Prove you are attached before believing a login wall.** A driver that fails + to attach does not error - it quietly drives its *own* fresh browser, and the + symptom is a logged-out page, which looks exactly like a real logged-out state. + `agent-browser connect ` is the known trap: on macOS it can relaunch a + local Chrome (`[agent-browser] relaunched browser`), so pass `--cdp` on every + command instead. To confirm you are on cuttle, check that the driver sees the + session's existing tabs (`playwright-cli attach` prints them) or that + `curl http://127.0.0.1:/json/version` names the same browser. +- **Leave the user's tabs alone.** The session is warm and shared - it may hold a + half-finished login or a page the user is watching in the viewer. Open your + work in a new tab rather than navigating the current one away, and close only + the tabs you opened. +- **One driver at a time.** Every client attached to a cuttle session shares one + browser and one set of tabs; two agents navigating in parallel clobber each + other. Serialize browser work, or give each worker its own identity with + `?fingerprint=` (see [Multi-seed farm](#multi-seed-farm)). +- **Read the site's API, not its DOM.** In a logged-in session the page already + carries the cookies and CSRF token, so an in-page `fetch()` of the site's own + JSON API (via the driver's `eval`) returns clean, complete data. Obfuscated or + lazily-hydrated class names make CSS-selector scraping report "element not + found" even when the content is on screen, and rendered text can silently + differ from what the site actually stored. +- **A logged-in session is the user's real account.** Reads (navigate, snapshot, + extract) are fine. Anything that writes - posting, commenting, reacting, + sending, purchasing, changing settings - needs the user's explicit go-ahead in + the current turn. Draft the content and hand it over; do not submit it. +- **Routing.** The briefing lists installed drivers in priority order; use the + first one (agent-browser by default) unless the user names another + (bu / bu-cli / browseruse = browser-use; playwright-cli). If the named driver + is not installed, use the first listed instead and tell the user you fell back. +- **Driver docs are fetched, not memorized.** Each driver self-documents at a + version-true source; the briefing gives the command per driver. Prefer the + full outputs (with templates/examples) over compact ones, and never rely on + a cached copy of another tool's docs. +- **No driver installed?** Stop and ask the user before installing anything. + Default offer: all three; minimal: just agent-browser. Drivers attach to + cuttle's browser, so skip their own browser downloads. + +Raw CDP libraries work too: + +```javascript +const browser = await chromium.connectOverCDP("http://127.0.0.1:9222"); +const page = browser.contexts()[0].pages()[0]; +``` + +`cuttle mcp [driver]` installs the CDP driver if absent and writes its MCP client +config pointed at the active context + profile, for agents that drive over MCP +rather than a shell. + +## Log into a site (VNC handoff) + +Log in **once** by hand; the profile keeps you logged in across restarts while a +CDP client drives the same live session. + +```bash +cuttle login https://accounts.google.com +# navigates there, opens the viewer, prints: open the viewer to sign in: http://127.0.0.1:6080/ +``` + +Open the viewer URL, sign in / solve the captcha there, and the CDP connection is +now logged in - VNC and CDP share one browser. This is why cuttle beats a fresh +headless browser for gated sites: the agent hits a wall, hands you the viewer +link, you sign in on the same session, nothing restarts. + +## Lifecycle + +```bash +cuttle status # container + CDP state +cuttle down # graceful stop (SIGTERM -> clean exit); KEEPS the profile +cuttle up # restart the stopped container - logins still there +cuttle down --purge # stop AND remove the container + discard the profile +cuttle up --recreate # destroy any existing container, start fresh +``` + +- **Graceful down matters.** `cuttle down` does `docker stop -t 15` so Chrome + exits clean; that avoids crash-restore junk tabs on next launch. Never + `docker rm -f` a running cuttle - the SIGKILL makes Chrome record a crash. +- **Profile = state.** Logins live in the container filesystem and survive + stop/start. `--purge` / `--recreate` are the only ways to discard them. +- `--keep-profile` (default on) is **fixed at container creation**; passing it (or + `--no-keep-profile`) against an existing container warns and is ignored - use + `--recreate` to change it. +- **`--image` and `--no-vnc` are creation-fixed too.** `--image` against an + existing container warns and is ignored (it will not switch a running container + to another image). `--no-vnc` is ignored *silently*: a container created with it + has no viewer port, yet a later plain `up` still prints a viewer URL that nothing + serves. `cuttle status` shows the container's real image and port bindings; + `--recreate` is the only way to change either, and it discards the profile. +- **Do not reach for `--recreate` on a port error.** If `up` says "container + restarted but CDP on : never came up", suspect a port mismatch first: a + restarted container keeps the ports it was *created* with, and `--recreate` + would discard the logged-in profile. Run `cuttle status` - it prints the real + port bindings and a log tail - then re-run `up` with those ports. + +Also on every subcommand: `--name` (run several side by side), `--no-vnc`, and on +`up` an `--image` override. `cuttle skill` prints this guide to stdout, always +matching the installed CLI. + +## Multi-seed farm + +For many isolated identities behind one endpoint - no CLI, no VNC - run the +container directly and select a seed per connection: + +```bash +docker run --rm -p 9222:9222 ghcr.io/glim-sh/cuttle:latest +``` + +``` +http://127.0.0.1:9222?fingerprint=12345 +http://127.0.0.1:9222?fingerprint=12345&timezone=America/New_York&locale=en-US +``` + +Each distinct `fingerprint` seed gets its own isolated Chrome with a stable, +coherent identity; point one CDP client per seed at the seed-parameterized URL. +**Proxy per seed:** pass an authenticated residential proxy on the connect URL - +cuttle strips the inline credentials and answers the proxy `407` over CDP, so +fork binaries that reject inline creds still work. Set proxy, `timezone`, and +`locale` together so the identity is coherent. A server-level default proxy for +every seed can be set with `CUTTLESERVE_PROXY`. + +## Engine swap + +Both forks are baked in, selected by `CLOAKBROWSER_BINARY_PATH`: + +```bash +docker run --rm -p 9222:9222 -e CLOAKBROWSER_BINARY_PATH=/opt/clearcote/chrome ghcr.io/glim-sh/cuttle:latest +``` + +- `/opt/clark/chrome` - clark, Chrome 148 (default) +- `/opt/clearcote/chrome` - clearcote, Chrome 149 (fallback) + +## Gotchas + +1. **Headed by default, on purpose.** The image runs `cuttle serve --headless=false` + on a built-in Xvfb; headed Chrome clears escalated anti-bot challenges headless + cannot. Do not force headless. +2. **VNC is loopback-only, no auth.** The viewer serves plain HTTP; the + `-p 127.0.0.1:PORT` mapping is the security boundary. Never bind it publicly. +3. **The engine stealth-presents an older version.** clark's binary is Chromium + 148 but reports `Chrome/146.x` over CDP/UA by design - a coherent, common + fingerprint, not a wrong build. Do not treat the reported version as a defect. +4. **"Logged in" can be false - but so can "logged out".** A CDP context may still + render a login form (geo often defaults to the egress country, which also drives + page *language* on logged-out pages - do not "fix" the language before checking + the signed-in page). The reverse trap is more common: a cookie read that returns + zero cookies is usually the driver probing its *own* blank page, not the site's + tab, while the session is perfectly alive. Verify auth by navigating the tab to + the site and checking what renders - and if the viewer shows you logged in, + trust the viewer. +5. **`cuttle view` is experimental** (CDP-screencast viewer instead of VNC): page + viewport only, no OAuth popups, frame delivery not yet reliable. Use the VNC + viewer (`cuttle up` / `cuttle login`) for real login flows. +6. **Sessions can be IP-bound.** A cookie minted at your real location, replayed + through a proxy in another geo, may force re-login + 2FA. Match the proxy geo + to where the session was created. +7. **"Container running but CDP not answering" / "restarted but CDP never came + up."** Usually a stale container from a *previous* `cuttle up` that failed + because the host port was taken (the failed run leaves a half-created + container with no live port binding). Current `cuttle up` auto-removes such + zombies and starts fresh; if you hit it on an older build, run + `cuttle up --recreate`. `cuttle status` prints a log tail with the real cause. +8. **One failed load is not a verdict on the browser.** Escalated challenges are + dominated by exit-IP reputation, not by the fingerprint: the same seed and + flags can clear in ~7s on a clean exit and take ~200s or fail on a flagged one. + If a page walls you, retry on a *new* identity (a different `?fingerprint=` + seed, and a different proxy exit if you use one) rather than hammering the same + one - do not conclude cuttle is broken from a single attempt. +9. **Chrome's container log noise is not a stealth failure.** `vkCreateInstance: + Found no drivers`, `Automatic fallback to software WebGL`, `Failed to connect to + the bus` (dbus), `Failed to adjust OOM score`, and `GPU stall due to ReadPixels` + are all expected on a headless host and do not mean the browser is broken or + detectable - a probe of the running seed returns a coherent spoofed GPU + (an ANGLE/Direct3D11 vendor-renderer pair), not SwiftShader. Do not "fix" them, + and never add `--enable-unsafe-swiftshader`: it exposes the raw software renderer + and makes the fingerprint worse. +10. **A crash on a `service_worker` target is a client bug, not detection.** Some + Chrome builds report a `service_worker` target without a `browserContextId`, and + older `playwright-core` asserts on it inside a CDP handler, killing the process + mid-run on a page that was loading fine. `cuttle serve` patches the target shape + so clients do not trip; if you drive cuttle with your own Playwright and still + hit it, pass `serviceWorkers: "block"` to `newContext`. Not a challenge failure. + +## Running on a server + +The amd64 image runs native on any Linux server: + +```bash +docker run -d --restart unless-stopped --name cuttle \ + -p 127.0.0.1:9222:9222 --shm-size=2g ghcr.io/glim-sh/cuttle:latest +``` + +Bind CDP to `127.0.0.1` and reach it over an SSH tunnel +(`ssh -L 9222:localhost:9222 user@server`) - the tunnel is the auth boundary, CDP +has none of its own. `--shm-size=2g` avoids Chrome crashes under load. Add +`-p 127.0.0.1:6080:6080 -e CUTTLE_VNC=1` for the viewer on the server too. The +`ssh` backend automates exactly this from the CLI. diff --git a/internal/cli/briefing.go b/internal/cli/briefing.go new file mode 100644 index 0000000..7b9e06c --- /dev/null +++ b/internal/cli/briefing.go @@ -0,0 +1,89 @@ +package cli + +import ( + "fmt" + "io" + "strconv" + "strings" +) + +// briefing is the single dynamic source of truth an agent needs to drive cuttle: +// live state plus installed drivers with attach lines and their own self-doc +// commands. cuttle carries no driver docs of its own. +type briefing struct { + verb string + location string // e.g. "container 'cuttle'" or "context 'cluster'" + imageTail string // ", image X" or "" + version string + cdpURL string + viewerURL string // "" = no viewer + engine string // browser string, "" = unknown + cdpPort int + drivers []detectedDriver +} + +func renderBriefing(w io.Writer, b briefing) { + engine := "" + if b.engine != "" { + engine = " (" + b.engine + ")" + } + fmt.Fprintf(w, "cuttle %s (%s%s) cuttle %s\n", b.verb, b.location, b.imageTail, b.version) + fmt.Fprintf(w, " CDP %s%s\n", b.cdpURL, engine) + if b.viewerURL != "" { + fmt.Fprintf(w, " viewer %s\n", b.viewerURL) + } + fmt.Fprintln(w) + fmt.Fprintln(w, "Attach to THIS browser over CDP. NEVER launch your own browser or create a") + fmt.Fprintln(w, "new profile/context: logins live in this one and persist across down/up.") + fmt.Fprintln(w) + + if len(b.drivers) > 0 { + fmt.Fprintln(w, "drivers (listed in priority order; the first is the default):") + for _, d := range b.drivers { + line := " " + d.name + if d.version != "" { + line += " " + d.version + } + fmt.Fprintln(w, line) + fmt.Fprintf(w, " attach %s\n", formatAttach(d.attach, b.cdpURL, b.cdpPort)) + fmt.Fprintf(w, " docs %s\n", d.docs) + } + for _, d := range drivers { + if !driverInstalled(b.drivers, d.name) { + fmt.Fprintf(w, " %s not installed (install: %s)\n", d.name, d.install) + } + } + fmt.Fprintln(w, "routing: use the first driver listed above unless the user names another") + fmt.Fprintln(w, " (bu / bu-cli / browseruse = browser-use). If the named driver is not") + fmt.Fprintln(w, " installed, use the first listed instead and tell the user you fell back.") + fmt.Fprintln(w, "docs: fetch each driver's own instructions with the `docs` command above -") + fmt.Fprintln(w, " they match the installed version; do not rely on memory or stale copies.") + } else { + fmt.Fprintln(w, "drivers: none installed. STOP and ask the user what to install -") + fmt.Fprintln(w, " default: all three; minimal: just agent-browser (the default driver).") + for _, d := range drivers { + fmt.Fprintf(w, " %s\n", d.install) + } + fmt.Fprintln(w, " (drivers attach to cuttle's browser - skip their own browser downloads)") + } + if b.viewerURL != "" { + fmt.Fprintln(w, "login walls / captcha: `cuttle login `, then hand the user the viewer") + fmt.Fprintln(w, " link to sign in or solve it - the CDP session stays logged in.") + } + fmt.Fprintln(w, "full cuttle guide: `cuttle skill` (prints the complete guide, always") + fmt.Fprintf(w, " matching this CLI %s; skip if you already loaded it this session)\n", b.version) +} + +func formatAttach(tmpl, cdpURL string, port int) string { + r := strings.NewReplacer("{cdp}", cdpURL, "{port}", strconv.Itoa(port)) + return r.Replace(tmpl) +} + +func driverInstalled(installed []detectedDriver, name string) bool { + for _, d := range installed { + if d.name == name { + return true + } + } + return false +} diff --git a/internal/cli/briefing_test.go b/internal/cli/briefing_test.go new file mode 100644 index 0000000..863ff0a --- /dev/null +++ b/internal/cli/briefing_test.go @@ -0,0 +1,113 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/glim-sh/cuttle/internal/backend" + "github.com/glim-sh/cuttle/internal/config" +) + +func TestRenderBriefingWithDrivers(t *testing.T) { + var sb strings.Builder + renderBriefing(&sb, briefing{ + verb: "ready", + location: "container 'cuttle'", + imageTail: ", image ghcr.io/glim-sh/cuttle:latest", + version: "0.3.0", + cdpURL: "http://127.0.0.1:9222", + viewerURL: "http://127.0.0.1:6080/", + engine: "Chrome/148", + cdpPort: 9222, + drivers: []detectedDriver{ + {driver: drivers[0], version: "0.31.1"}, // agent-browser installed + }, + }) + out := sb.String() + + wantContains := []string{ + "cuttle ready (container 'cuttle', image ghcr.io/glim-sh/cuttle:latest) cuttle 0.3.0", + "CDP http://127.0.0.1:9222 (Chrome/148)", + "viewer http://127.0.0.1:6080/", + "agent-browser 0.31.1", + "attach agent-browser --cdp 9222 ", + "browser-use not installed (install: uv tool install browser-use)", + "playwright-cli not installed", + "login walls / captcha: `cuttle login `", + } + for _, w := range wantContains { + if !strings.Contains(out, w) { + t.Fatalf("briefing missing %q\n---\n%s", w, out) + } + } +} + +func TestRenderBriefingNoDrivers(t *testing.T) { + var sb strings.Builder + renderBriefing(&sb, briefing{ + verb: "ready", location: "context 'cluster'", version: "0.3.0", + cdpURL: "http://127.0.0.1:40001", cdpPort: 40001, + }) + out := sb.String() + if !strings.Contains(out, "drivers: none installed. STOP and ask the user") { + t.Fatalf("expected none-installed branch:\n%s", out) + } + // no viewer line, and no login-wall hint when there is no viewer + if strings.Contains(out, "viewer ") || strings.Contains(out, "login walls") { + t.Fatalf("viewerless briefing should omit viewer/login hints:\n%s", out) + } + for _, d := range drivers { + if !strings.Contains(out, d.install) { + t.Fatalf("missing install hint %q", d.install) + } + } +} + +func TestFormatAttach(t *testing.T) { + tests := []struct { + tmpl string + want string + }{ + {"agent-browser --cdp {port} ", "agent-browser --cdp 8080 "}, + {"BU_CDP_URL={cdp} browser-use", "BU_CDP_URL=http://127.0.0.1:8080 browser-use"}, + {"playwright-cli attach --cdp={cdp}", "playwright-cli attach --cdp=http://127.0.0.1:8080"}, + } + for _, tt := range tests { + if got := formatAttach(tt.tmpl, "http://127.0.0.1:8080", 8080); got != tt.want { + t.Fatalf("formatAttach(%q)=%q want %q", tt.tmpl, got, tt.want) + } + } +} + +func TestBoolFlagOptional(t *testing.T) { + var b boolFlag + if b.value() != nil { + t.Fatal("unset boolFlag should be nil") + } + if err := b.Set("true"); err != nil { + t.Fatalf("Set: %v", err) + } + if v := b.value(); v == nil || !*v { + t.Fatalf("want true, got %v", v) + } +} + +func TestEndpointURLs(t *testing.T) { + ep := backend.Endpoint{CDPHost: "127.0.0.1", CDPPort: 9222, VNCHost: "127.0.0.1", VNCPort: 6080} + cdp, viewer := endpointURLs(ep, false) + if cdp != "http://127.0.0.1:9222" || viewer != "http://127.0.0.1:6080/" { + t.Fatalf("urls: %q %q", cdp, viewer) + } + if _, viewer := endpointURLs(ep, true); viewer != "" { + t.Fatalf("no-vnc should suppress viewer, got %q", viewer) + } +} + +func TestLocationLabel(t *testing.T) { + if got := locationLabel("local", config.Context{Backend: config.BackendLocal}, "cuttle"); got != "container 'cuttle'" { + t.Fatalf("local label: %q", got) + } + if got := locationLabel("cluster", config.Context{Backend: config.BackendK8s}, "cuttle"); got != "context 'cluster'" { + t.Fatalf("remote label: %q", got) + } +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go new file mode 100644 index 0000000..4d39c40 --- /dev/null +++ b/internal/cli/commands.go @@ -0,0 +1,665 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "os/signal" + "runtime" + "strconv" + "syscall" + "time" + + "github.com/spf13/cobra" + + "github.com/glim-sh/cuttle/internal/backend" + "github.com/glim-sh/cuttle/internal/config" + "github.com/glim-sh/cuttle/internal/profile" +) + +// boolFlag is an optional bool: unset (nil) is distinct from explicit +// true/false, so up can warn that --keep-profile is fixed at container creation. +// `--keep-profile` sets true (NoOptDefVal); `--keep-profile=false` sets false. +type boolFlag struct { + set bool + val bool +} + +func (b *boolFlag) String() string { + if b.val { + return "true" + } + return "false" +} + +func (b *boolFlag) Set(s string) error { + v, err := strconv.ParseBool(s) + if err != nil { + return err //nolint:wrapcheck // pflag renders it + } + b.val, b.set = v, true + return nil +} + +func (b *boolFlag) Type() string { return "bool" } + +func (b *boolFlag) value() *bool { + if !b.set { + return nil + } + return &b.val +} + +const ( + defaultName = "cuttle" + defaultCDPPort = 9222 + defaultVNCPort = 6080 + imageRepo = "ghcr.io/glim-sh/cuttle" +) + +var ( + errCDPNotAnswering = errors.New("CDP not answering - run `cuttle up` first") + errNoContainer = errors.New("no container (run `cuttle up`)") +) + +func init() { + AddCommand(newUpCmd(), newDownCmd(), newStatusCmd(), newLoginCmd(), newConnectCmd(), newMCPCmd(), newContextCmd()) +} + +// defaultImage is the published image tag matching this CLI's version, so it +// never drives a cuttleserve it was not shipped with. An uninstalled checkout +// reports "dev" (no such tag), so it falls back to latest. +func defaultImage() string { + v := version + if v == "dev" { + v = "latest" + } + return imageRepo + ":" + v +} + +type commonFlags struct { + contextName string + name string + cdpPort int + vncPort int + noVNC bool + profile string // seed; set only on verbs that take --profile (login/connect/mcp) +} + +func addCommonFlags(cmd *cobra.Command, cf *commonFlags) { + f := cmd.Flags() + f.StringVar(&cf.contextName, "context", "", "context to use (default: config default_context, else local)") + f.StringVar(&cf.name, "name", defaultName, "container name") + f.IntVar(&cf.cdpPort, "cdp-port", defaultCDPPort, "host CDP port") + f.IntVar(&cf.vncPort, "vnc-port", defaultVNCPort, "host VNC viewer port") + f.BoolVar(&cf.noVNC, "no-vnc", false, "run without the VNC viewer") +} + +// addProfileFlag wires --profile on the verbs that drive a session with a named +// profile (its local auth state is checked out for the session duration). +func addProfileFlag(cmd *cobra.Command, p *string) { + cmd.Flags().StringVar(p, "profile", "", "profile name (= seed); checks out its local auth state for this session") +} + +// withFingerprint appends the profile as a ?fingerprint= query so a driver +// attaching at this URL lands on the profile's seed. +func withFingerprint(base, profileName string) string { + if profileName == "" { + return base + } + return base + "?fingerprint=" + url.QueryEscape(profileName) +} + +// profileRemote reports whether the named profile is configured as remote- +// persistent storage (no checkout/checkin). An unconfigured profile is local. +func profileRemote(name string) (bool, error) { + if name == "" { + return false, nil + } + cfg, err := config.Load() + if err != nil { + return false, err + } + p, ok := cfg.Profiles[name] + return ok && p.Storage == config.StorageRemote, nil +} + +// checkoutProfile starts a profile session against the reachable endpoint when +// --profile is set, returning nil when it is not. The caller MUST Close the +// returned session (via defer and a signal-aware context) to check state in. +func checkoutProfile(ctx context.Context, cf commonFlags, ep backend.Endpoint) (*profile.Session, error) { + if cf.profile == "" { + return nil, nil + } + if !profile.ValidName(cf.profile) { + return nil, fmt.Errorf("%w: %q", errInvalidProfile, cf.profile) + } + remote, err := profileRemote(cf.profile) + if err != nil { + return nil, err + } + base := "http://" + net.JoinHostPort(ep.CDPHost, strconv.Itoa(ep.CDPPort)) + return profile.Checkout(ctx, profile.Options{Name: cf.profile, CDPBase: base, Remote: remote}) +} + +var errInvalidProfile = errors.New("invalid profile name") + +// resolve loads the config, selects the active context, and builds its backend. +// For the docker-container backends (local, ssh) the container is named by the +// --name flag (default "cuttle", matching the Python CLI); k8s/direct ignore it +// and are identified by their context. +func resolve(cf commonFlags, image string) (string, config.Context, backend.Backend, error) { + cfg, err := config.Load() + if err != nil { + return "", config.Context{}, nil, err + } + ctxName, ctx, err := cfg.Active(cf.contextName, os.Getenv(config.EnvContext)) + if err != nil { + return "", config.Context{}, nil, err + } + name := ctxName + if ctx.Backend == config.BackendLocal || ctx.Backend == config.BackendSSH { + name = cf.name + } + b, err := backend.New(name, ctx, backend.ExecRunner{}, cf.cdpPort, cf.vncPort, image) + if err != nil { + return "", config.Context{}, nil, err + } + return name, ctx, b, nil +} + +func locationLabel(ctxName string, ctx config.Context, name string) string { + if ctx.Backend == config.BackendLocal || ctx.Backend == "" { + return "container '" + name + "'" + } + return "context '" + ctxName + "'" +} + +func endpointURLs(ep backend.Endpoint, noVNC bool) (string, string) { + cdp := "http://" + net.JoinHostPort(ep.CDPHost, strconv.Itoa(ep.CDPPort)) + viewer := "" + if !noVNC && ep.VNCPort != 0 { + viewer = "http://" + net.JoinHostPort(ep.VNCHost, strconv.Itoa(ep.VNCPort)) + "/" + } + return cdp, viewer +} + +func cdpReady(ctx context.Context, host string, port int, timeout time.Duration) map[string]any { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + endpoint := "http://" + net.JoinHostPort(host, strconv.Itoa(port)) + "/json/version" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil + } + var v map[string]any + if json.Unmarshal(body, &v) != nil { + return nil + } + return v +} + +func waitCDP(ctx context.Context, host string, port int, timeout time.Duration) map[string]any { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if v := cdpReady(ctx, host, port, 500*time.Millisecond); v != nil { + return v + } + select { + case <-ctx.Done(): + return nil + case <-time.After(500 * time.Millisecond): + } + } + return nil +} + +func browserOf(v map[string]any) string { + if v == nil { + return "" + } + b, _ := v["Browser"].(string) + return b +} + +// --------------------------------------------------------------------------- +// up +// --------------------------------------------------------------------------- + +type upFlags struct { + common commonFlags + image string + keepProfile boolFlag + recreate bool +} + +func newUpCmd() *cobra.Command { + var uf upFlags + cmd := &cobra.Command{ + Use: "up", + Short: "start the browser (idempotent) with VNC viewing", + RunE: func(cmd *cobra.Command, _ []string) error { return runUp(cmd, &uf) }, + } + addCommonFlags(cmd, &uf.common) + cmd.Flags().StringVar(&uf.image, "image", "", "image (default "+defaultImage()+"; use cuttle:local for a local build)") + cmd.Flags().Var(&uf.keepProfile, "keep-profile", "persist the browser profile across restarts (default on)") + cmd.Flags().Lookup("keep-profile").NoOptDefVal = "true" + cmd.Flags().BoolVar(&uf.recreate, "recreate", false, "destroy any existing container and start fresh (discards the profile)") + return cmd +} + +func runUp(cmd *cobra.Command, uf *upFlags) error { + name, ctx, b, err := resolve(uf.common, defaultImage()) + if err != nil { + return err + } + before, err := b.State(cmd.Context()) + if err != nil { + return err + } + + if before != backend.StateAbsent { + if uf.image != "" { + fmt.Fprintf(os.Stderr, "cuttle: --image is fixed when the container is created; %q keeps the image it was created with (use --recreate to change it)\n", name) + } + if uf.keepProfile.set { + fmt.Fprintf(os.Stderr, "cuttle: --keep-profile is fixed when the container is created; %q keeps its original setting (use --recreate to change it)\n", name) + } + } + + opts := backend.StartOpts{ + Image: uf.image, + Recreate: uf.recreate, + KeepProfile: uf.keepProfile.value(), + NoVNC: uf.common.noVNC, + Proxy: ctx.Proxy, + Storage: config.StorageLocal, + } + if err = b.Start(cmd.Context(), opts); err != nil { + return err + } + + ep, release, err := b.Reach(cmd.Context(), 0, 0) + if err != nil { + return err + } + defer release() + + v := waitCDP(cmd.Context(), ep.CDPHost, ep.CDPPort, 30*time.Second) + if v == nil { + if before == backend.StateRunning { + return fmt.Errorf("%q is running but CDP is not answering - run `cuttle status` to triage, then `cuttle down` and retry", name) //nolint:err113 + } + return errors.New("started but CDP never came up - run `cuttle status` to triage") //nolint:err113 + } + + verb, showImage := "ready", true + switch before { + case backend.StateRunning: + verb, showImage = "already running", false + case backend.StateStopped: + verb, showImage = "restarted", false + case backend.StateAbsent: + } + image := uf.image + if image == "" { + image = defaultImage() + } + printBriefingFor(cmd.OutOrStdout(), verb, name, ctx, uf.common, ep, browserOf(v), image, showImage) + return nil +} + +func printBriefingFor(w io.Writer, verb, name string, ctx config.Context, cf commonFlags, ep backend.Endpoint, engine, image string, showImage bool) { + cdp, viewer := endpointURLs(ep, cf.noVNC) + cdp = withFingerprint(cdp, cf.profile) + imageTail := "" + if showImage && (ctx.Backend == config.BackendLocal || ctx.Backend == "") { + imageTail = ", image " + image + } + renderBriefing(w, briefing{ + verb: verb, + location: locationLabel(cf.contextName, ctx, name), + imageTail: imageTail, + version: version, + cdpURL: cdp, + viewerURL: viewer, + engine: engine, + cdpPort: ep.CDPPort, + drivers: detectDrivers(), + }) +} + +// --------------------------------------------------------------------------- +// down +// --------------------------------------------------------------------------- + +func newDownCmd() *cobra.Command { + var cf commonFlags + var purge bool + cmd := &cobra.Command{ + Use: "down", + Short: "stop the browser gracefully (keeps the profile)", + RunE: func(cmd *cobra.Command, _ []string) error { + name, ctx, b, err := resolve(cf, defaultImage()) + if err != nil { + return err + } + state, err := b.State(cmd.Context()) + if err != nil { + return err + } + if state == backend.StateAbsent { + fmt.Fprintf(cmd.OutOrStdout(), "cuttle: nothing to stop (%s)\n", locationLabel(cf.contextName, ctx, name)) + return nil + } + if err := b.Stop(cmd.Context(), purge); err != nil { + return err + } + if purge { + fmt.Fprintf(cmd.OutOrStdout(), "cuttle: removed %s (profile discarded)\n", locationLabel(cf.contextName, ctx, name)) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "cuttle: stopped %s (profile kept; `cuttle up` to resume)\n", locationLabel(cf.contextName, ctx, name)) + } + return nil + }, + } + addCommonFlags(cmd, &cf) + cmd.Flags().BoolVar(&purge, "purge", false, "also remove the container/release and discard the profile") + return cmd +} + +// --------------------------------------------------------------------------- +// status +// --------------------------------------------------------------------------- + +func newStatusCmd() *cobra.Command { + var cf commonFlags + cmd := &cobra.Command{ + Use: "status", + Short: "show browser + CDP state", + RunE: func(cmd *cobra.Command, _ []string) error { return runStatus(cmd, cf) }, + } + addCommonFlags(cmd, &cf) + return cmd +} + +func runStatus(cmd *cobra.Command, cf commonFlags) error { + name, ctx, b, err := resolve(cf, defaultImage()) + if err != nil { + return err + } + out := cmd.OutOrStdout() + state, err := b.State(cmd.Context()) + if err != nil { + return err + } + if state == backend.StateAbsent { + return errNoContainer + } + + ep, release, err := b.Reach(cmd.Context(), 0, 0) + if err != nil { + return err + } + defer release() + + v := waitCDP(cmd.Context(), ep.CDPHost, ep.CDPPort, 5*time.Second) + if state == backend.StateRunning && v != nil { + printBriefingFor(out, "running", name, ctx, cf, ep, browserOf(v), "", false) + if img := localImage(cmd.Context(), b); img != "" { + fmt.Fprintf(out, " image %s\n", img) + } + return nil + } + + cdp, viewer := endpointURLs(ep, cf.noVNC) + fmt.Fprintf(out, "%s: %s\n", locationLabel(cf.contextName, ctx, name), state) + if v == nil { + fmt.Fprintf(out, " CDP %s (not answering)\n", cdp) + } else { + fmt.Fprintf(out, " CDP %s\n", cdp) + } + if viewer != "" { + fmt.Fprintf(out, " viewer %s\n", viewer) + } + if img := localImage(cmd.Context(), b); img != "" { + fmt.Fprintf(out, " image %s\n", img) + } + if d, ok := b.(interface { + Diagnostics(context.Context) []string + }); ok { + for _, line := range d.Diagnostics(cmd.Context()) { + fmt.Fprintf(out, " %s\n", line) + } + } + fmt.Fprintln(out, " fix: `cuttle down && cuttle up` (keeps the profile), or") + fmt.Fprintln(out, " `cuttle up --recreate` to rebuild from scratch (discards the profile).") + return errUnhealthy +} + +var errUnhealthy = errors.New("browser unhealthy") + +func localImage(ctx context.Context, b backend.Backend) string { + if im, ok := b.(interface { + Image(context.Context) string + }); ok { + return im.Image(ctx) + } + return "" +} + +// --------------------------------------------------------------------------- +// login +// --------------------------------------------------------------------------- + +func newLoginCmd() *cobra.Command { + var cf commonFlags + var noOpen bool + cmd := &cobra.Command{ + Use: "login ", + Short: "navigate to a URL and open the viewer to sign in", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { return runLogin(cmd, cf, args[0], noOpen) }, + } + addCommonFlags(cmd, &cf) + addProfileFlag(cmd, &cf.profile) + cmd.Flags().BoolVar(&noOpen, "no-open", false, "print the viewer URL, don't open it") + return cmd +} + +func runLogin(cmd *cobra.Command, cf commonFlags, target string, noOpen bool) error { + _, _, b, err := resolve(cf, defaultImage()) + if err != nil { + return err + } + ep, release, err := b.Reach(cmd.Context(), 0, 0) + if err != nil { + return err + } + defer release() + + if cdpReady(cmd.Context(), ep.CDPHost, ep.CDPPort, 5*time.Second) == nil { + return errCDPNotAnswering + } + + // Install the signal handler before checkout so a Ctrl-C during or right + // after checkout still runs the deferred Close (checkin + lock release). + sigCtx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // With --profile, check the local auth state into the seed before navigating, + // so the user signs in on top of any prior session, and check it back in on + // exit (including Ctrl-C) so the fresh login is captured. + sess, err := checkoutProfile(cmd.Context(), cf, ep) + if err != nil { + return err + } + if sess != nil { + defer func() { _ = sess.Close() }() + } + + vncPort := 0 + if !cf.noVNC { + vncPort = ep.VNCPort + } + title, err := navigate(cmd.Context(), ep.CDPHost, ep.CDPPort, target, vncPort, cf.profile) + if err != nil { + return fmt.Errorf("navigation failed: %w", err) + } + out := cmd.OutOrStdout() + line := "navigated to " + target + if title != "" { + line += " (" + title + ")" + } + fmt.Fprintln(out, line) + _, viewer := endpointURLs(ep, cf.noVNC) + if viewer != "" { + fmt.Fprintf(out, "open the viewer to sign in: %s\n", viewer) + if !noOpen { + openBrowser(viewer) + } + } + + if sess != nil { + fmt.Fprintln(out, "profile checked out - sign in via the viewer, then press Ctrl-C to save the session.") + <-sigCtx.Done() + fmt.Fprintln(out, "\ncuttle: saving profile state...") + } + return nil +} + +// --------------------------------------------------------------------------- +// connect +// --------------------------------------------------------------------------- + +func newConnectCmd() *cobra.Command { + var cf commonFlags + cmd := &cobra.Command{ + Use: "connect", + Short: "hold a forward open in the foreground and print the driver briefing (Ctrl-C to end)", + RunE: func(cmd *cobra.Command, _ []string) error { return runConnect(cmd, cf) }, + } + addCommonFlags(cmd, &cf) + addProfileFlag(cmd, &cf.profile) + return cmd +} + +func runConnect(cmd *cobra.Command, cf commonFlags) error { + name, ctx, b, err := resolve(cf, defaultImage()) + if err != nil { + return err + } + // connect holds the forward open for a driver to attach to, so it pins the + // local ports (matching what `cuttle mcp` writes); the ephemeral commands + // auto-pick instead. + ep, release, err := b.Reach(cmd.Context(), cf.cdpPort, cf.vncPort) + if err != nil { + return err + } + defer release() + + v := waitCDP(cmd.Context(), ep.CDPHost, ep.CDPPort, 30*time.Second) + if v == nil { + return errCDPNotAnswering + } + + // Install the signal handler before checkout so a Ctrl-C during or right + // after checkout still runs the deferred Close (checkin + lock release) + // instead of terminating the process with the profile left locked. + sigCtx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + sess, err := checkoutProfile(cmd.Context(), cf, ep) + if err != nil { + return err + } + if sess != nil { + defer func() { _ = sess.Close() }() + } + + out := cmd.OutOrStdout() + printBriefingFor(out, "connected", name, ctx, cf, ep, browserOf(v), "", false) + fmt.Fprintln(out, "forward held open - press Ctrl-C to end the session.") + + <-sigCtx.Done() + if sess != nil { + fmt.Fprintln(out, "\ncuttle: saving profile state...") + } else { + fmt.Fprintln(out, "\ncuttle: forward closed.") + } + return nil +} + +// --------------------------------------------------------------------------- +// context ls +// --------------------------------------------------------------------------- + +func newContextCmd() *cobra.Command { + cmd := &cobra.Command{Use: "context", Short: "inspect cuttle contexts"} + var contextName string + ls := &cobra.Command{ + Use: "ls", + Short: "list contexts and show the active one", + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + active, _, err := cfg.Active(contextName, os.Getenv(config.EnvContext)) + if err != nil { + return err + } + out := cmd.OutOrStdout() + for _, n := range cfg.Names() { + marker := " " + if n == active { + marker = "* " + } + fmt.Fprintf(out, "%s%-16s %s\n", marker, n, cfg.Contexts[n].Backend) + } + return nil + }, + } + ls.Flags().StringVar(&contextName, "context", "", "context to mark active (default: config default_context, else local)") + cmd.AddCommand(ls) + return cmd +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +// openBrowser best-effort opens a URL in the user's default browser. +func openBrowser(link string) { + var name string + var args []string + switch runtime.GOOS { + case "darwin": + name = "open" + args = []string{link} + case "windows": + name, args = "rundll32", []string{"url.dll,FileProtocolHandler", link} + default: + name, args = "xdg-open", []string{link} + } + if _, err := exec.LookPath(name); err != nil { + return + } + _ = exec.CommandContext(context.Background(), name, args...).Start() +} diff --git a/internal/cli/drivers.go b/internal/cli/drivers.go new file mode 100644 index 0000000..aae9eee --- /dev/null +++ b/internal/cli/drivers.go @@ -0,0 +1,116 @@ +package cli + +import ( + "context" + "os" + "os/exec" + "strings" + "sync" + "time" +) + +// driver is a CDP driver CLI cuttle knows how to route agents to. cuttle never +// bakes in driver documentation - each driver self-documents at runtime (docs), +// so instructions always match the installed version. The briefing only carries +// the attach incantation and where the docs live. +type driver struct { + name string + attach string // {cdp} = http endpoint, {port} = CDP port + docs string + install string + // nil = never probe: browser-use treats unknown argv as harness input and + // would launch its daemon from a mere version check. + versionArgs []string +} + +// Briefing order IS the fallback order: the first installed entry is the default. +var drivers = []driver{ + { + name: "agent-browser", + attach: "agent-browser --cdp {port} # --cdp on EVERY command; never `connect`", + docs: "agent-browser skills get core --full", + install: "npm install -g agent-browser", + versionArgs: []string{"--version"}, + }, + { + name: "browser-use", + attach: "BU_CDP_URL={cdp} browser-use <<'PY' ... PY", + docs: "browser-use skill show", + install: "uv tool install browser-use", + versionArgs: nil, + }, + { + name: "playwright-cli", + attach: "playwright-cli attach --cdp={cdp}", + docs: "playwright-cli --help # its 'Agent skill:' line -> full SKILL.md + references/", + install: "npm install -g @playwright/cli", + versionArgs: []string{"--version"}, + }, +} + +type detectedDriver struct { + driver + version string // "" when unknown or not probed +} + +// detectDrivers returns the installed drivers in briefing order, each with its +// version when cheaply knowable. Version probes run in parallel so the briefing +// stays fast; a probe failure degrades to a versionless line, never an error. +func detectDrivers() []detectedDriver { + type found struct { + d driver + exe string + } + var installed []found + for _, d := range drivers { + if exe, err := exec.LookPath(d.name); err == nil { + installed = append(installed, found{d, exe}) + } + } + if len(installed) == 0 { + return nil + } + + versions := make([]string, len(installed)) + var wg sync.WaitGroup + for i, f := range installed { + if f.d.versionArgs == nil { + continue + } + wg.Go(func() { + versions[i] = driverVersion(f.d.name, f.exe, f.d.versionArgs) + }) + } + wg.Wait() + + out := make([]detectedDriver, len(installed)) + for i, f := range installed { + out[i] = detectedDriver{driver: f.d, version: versions[i]} + } + return out +} + +func driverVersion(name, exe string, versionArgs []string) string { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, exe, versionArgs...) + cmd.Env = append(os.Environ(), "NO_UPDATE_NOTIFIER=1") + out, err := cmd.CombinedOutput() + if err != nil && len(out) == 0 { + return "" + } + first := "" + for line := range strings.SplitSeq(string(out), "\n") { + if s := strings.TrimSpace(line); s != "" { + first = s + break + } + } + // Some drivers echo their own name ("agent-browser 0.31.1"); the briefing + // already prints the name, so keep just the version. + v := strings.TrimSpace(strings.TrimPrefix(first, name)) + if len(v) > 40 { + v = v[:40] + } + return v +} diff --git a/internal/cli/mcp.go b/internal/cli/mcp.go new file mode 100644 index 0000000..b1ec04c --- /dev/null +++ b/internal/cli/mcp.go @@ -0,0 +1,83 @@ +package cli + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" + + "github.com/glim-sh/cuttle/internal/config" + "github.com/glim-sh/cuttle/internal/mcp" + "github.com/glim-sh/cuttle/internal/profile" +) + +func newMCPCmd() *cobra.Command { + var cf commonFlags + var output string + cmd := &cobra.Command{ + Use: "mcp [driver]", + Short: "install a CDP driver and write its MCP config pointed at this context + profile", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { return runMCP(cmd, cf, args, output) }, + } + addCommonFlags(cmd, &cf) + addProfileFlag(cmd, &cf.profile) + cmd.Flags().StringVar(&output, "output", "", "write the config here ('-' = stdout; default $XDG_CONFIG_HOME/cuttle/mcp/.json)") + return cmd +} + +func runMCP(cmd *cobra.Command, cf commonFlags, args []string, output string) error { + driverName := mcp.DefaultDriver + if len(args) == 1 { + driverName = args[0] + } + d, err := mcp.Lookup(driverName) + if err != nil { + return err + } + if cf.profile != "" && !profile.ValidName(cf.profile) { + return fmt.Errorf("%w: %q", errInvalidProfile, cf.profile) + } + + _, ctx, _, err := resolve(cf, defaultImage()) + if err != nil { + return err + } + cdpURL := mcpCDPURL(ctx, cf) + + if ierr := mcp.EnsureInstalled(cmd.Context(), d); ierr != nil { + return ierr + } + data, err := mcp.Marshal(mcp.BuildConfig(d, cdpURL)) + if err != nil { + return err + } + + out := cmd.OutOrStdout() + if output == "-" { + fmt.Fprintln(out, string(data)) + return nil + } + path := output + if path == "" { + path = mcp.DefaultConfigPath(d.Name) + } + if err := mcp.WriteConfig(path, data); err != nil { + return err + } + fmt.Fprintf(out, "wrote %s MCP config -> %s\n", d.Name, path) + fmt.Fprintf(out, " CDP %s\n", cdpURL) + return nil +} + +// mcpCDPURL is the CDP endpoint the driver attaches to: the configured URL for a +// direct context, else the conventional loopback CDP port a local container or a +// held `cuttle connect` forward exposes. The profile seed is appended as +// ?fingerprint so the driver lands on the profile's browser. +func mcpCDPURL(ctx config.Context, cf commonFlags) string { + base := "http://127.0.0.1:" + strconv.Itoa(cf.cdpPort) + if ctx.Backend == config.BackendDirect && ctx.CDPURL != "" { + base = ctx.CDPURL + } + return withFingerprint(base, cf.profile) +} diff --git a/internal/cli/navigate.go b/internal/cli/navigate.go new file mode 100644 index 0000000..fbd061e --- /dev/null +++ b/internal/cli/navigate.go @@ -0,0 +1,158 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/coder/websocket" +) + +// navigate points the browser's active page at url over CDP and returns its +// title (best effort). It lists targets via the multiplexer's /json endpoint, +// picks the page a human would be driving, and issues Page.navigate on the +// per-page WebSocket the list hands back. +func navigate(ctx context.Context, host string, port int, targetURL string, vncPort int, seed string) (string, error) { + targets, err := listTargets(ctx, host, port, seed) + if err != nil { + return "", err + } + target := pickPage(targets, vncPort) + wsURL, _ := target["webSocketDebuggerUrl"].(string) + if wsURL == "" { + return "", errNoPageTarget + } + + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + conn, resp, err := websocket.Dial(ctx, wsURL, nil) + if resp != nil && resp.Body != nil { + _ = resp.Body.Close() + } + if err != nil { + return "", fmt.Errorf("connecting to CDP page: %w", err) + } + defer func() { _ = conn.Close(websocket.StatusNormalClosure, "") }() + + s := &cdpSession{conn: conn} + if _, cerr := s.call(ctx, "Page.navigate", map[string]any{"url": targetURL}); cerr != nil { + return "", cerr + } + res, err := s.call(ctx, "Runtime.evaluate", map[string]any{ + "expression": "document.title", "returnByValue": true, + }) + if err != nil { + return "", nil + } + if r, ok := res["result"].(map[string]any); ok { + if v, ok := r["value"].(string); ok { + return v, nil + } + } + return "", nil +} + +var ( + errNoPageTarget = errors.New("no page target found to navigate") + errCDPError = errors.New("CDP error") +) + +func listTargets(ctx context.Context, host string, port int, seed string) ([]map[string]any, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + endpoint := withFingerprint("http://"+net.JoinHostPort(host, strconv.Itoa(port))+"/json", seed) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err //nolint:wrapcheck + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("listing CDP targets: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err //nolint:wrapcheck + } + var targets []map[string]any + if err := json.Unmarshal(body, &targets); err != nil { + return nil, err //nolint:wrapcheck + } + return targets, nil +} + +// pickPage chooses the page target a human is most likely driving, skipping +// worker/service targets, the built-in tab-search UI (chrome://), and the local +// VNC viewer page itself. +func pickPage(targets []map[string]any, vncPort int) map[string]any { + viewerPrefix := "" + if vncPort != 0 { + viewerPrefix = "http://127.0.0.1:" + strconv.Itoa(vncPort) + "/" + } + var pages []map[string]any + for _, t := range targets { + if ty, _ := t["type"].(string); ty == "page" { + pages = append(pages, t) + } + } + for _, t := range pages { + u, _ := t["url"].(string) + if strings.HasPrefix(u, "chrome://") { + continue + } + if viewerPrefix != "" && strings.HasPrefix(u, viewerPrefix) { + continue + } + return t + } + if len(pages) > 0 { + return pages[0] + } + return nil +} + +// cdpSession is a single WebSocket connection to one CDP target with id-matched +// calls. +type cdpSession struct { + conn *websocket.Conn + nextID int +} + +func (s *cdpSession) call(ctx context.Context, method string, params map[string]any) (map[string]any, error) { + s.nextID++ + id := s.nextID + payload, err := json.Marshal(map[string]any{"id": id, "method": method, "params": params}) + if err != nil { + return nil, err //nolint:wrapcheck + } + if err := s.conn.Write(ctx, websocket.MessageText, payload); err != nil { + return nil, fmt.Errorf("CDP write: %w", err) + } + for { + _, data, err := s.conn.Read(ctx) + if err != nil { + return nil, fmt.Errorf("CDP read: %w", err) + } + var msg map[string]any + if err := json.Unmarshal(data, &msg); err != nil { + continue + } + mid, ok := msg["id"].(float64) + if !ok || int(mid) != id { + continue + } + if e, ok := msg["error"].(map[string]any); ok { + m, _ := e["message"].(string) + return nil, fmt.Errorf("%w: %s: %s", errCDPError, method, m) + } + result, _ := msg["result"].(map[string]any) + return result, nil + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go new file mode 100644 index 0000000..384d53a --- /dev/null +++ b/internal/cli/root.go @@ -0,0 +1,34 @@ +// Package cli wires the cuttle command tree. Subcommands are registered on the +// root command by later phases via [AddCommand]. +package cli + +import ( + "github.com/spf13/cobra" +) + +// version is the CLI version, overridden at build time via -ldflags. +var version = "dev" + +func newRootCmd() *cobra.Command { + root := &cobra.Command{ + Use: "cuttle", + Short: "Stealth-Chromium browser farm CLI", + Version: version, + SilenceUsage: true, + SilenceErrors: true, + } + return root +} + +var rootCmd = newRootCmd() + +// AddCommand registers a subcommand on the root command. Later phases call this +// from their package init to plug in verbs (up/down/status/serve/...). +func AddCommand(cmds ...*cobra.Command) { + rootCmd.AddCommand(cmds...) +} + +// Execute runs the root command and returns its error for main to report. +func Execute() error { + return rootCmd.Execute() //nolint:wrapcheck // cobra prints the error itself +} diff --git a/internal/cli/skill.go b/internal/cli/skill.go new file mode 100644 index 0000000..27a91f4 --- /dev/null +++ b/internal/cli/skill.go @@ -0,0 +1,31 @@ +package cli + +import ( + _ "embed" + + "github.com/spf13/cobra" +) + +// skillGuide is the full agent-facing cuttle guide, compiled into the binary so +// `cuttle skill` always prints the doc that matches this CLI. The repo-root +// SKILL.md is a symlink to this file (go:embed cannot reach parent dirs or +// follow symlinks, so the real file lives in-package). +// +//go:embed SKILL.md +var skillGuide string + +func init() { + AddCommand(newSkillCmd()) +} + +func newSkillCmd() *cobra.Command { + return &cobra.Command{ + Use: "skill", + Short: "print the full agent-facing cuttle guide", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + _, err := cmd.OutOrStdout().Write([]byte(skillGuide)) + return err //nolint:wrapcheck // cobra prints the error itself + }, + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..01d755e --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,165 @@ +// Package config loads the cuttle TOML config and resolves the active context. +// +// The config is read-mostly: a single file at $XDG_CONFIG_HOME/cuttle/config.toml +// declares named contexts (where the browser runs) and named profiles (= seeds). +// A missing file yields a built-in "local" context, so cuttle needs zero config +// to drive a local docker browser. +package config + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + + toml "github.com/pelletier/go-toml/v2" + + "github.com/glim-sh/cuttle/internal/xdg" +) + +// Backend names selectable per context. +const ( + BackendLocal = "local" + BackendK8s = "k8s" + BackendSSH = "ssh" + BackendDirect = "direct" +) + +// Storage modes for a profile. +const ( + StorageLocal = "local" + StorageRemote = "remote" +) + +// EnvContext is the environment variable that selects the active context. +const EnvContext = "CUTTLE_CONTEXT" + +// Config is the parsed config file. Missing keys leave zero values. +type Config struct { + DefaultContext string `toml:"default_context"` + Contexts map[string]Context `toml:"context"` + Profiles map[string]Profile `toml:"profile"` +} + +// Context describes where and how a browser runs. Which fields are meaningful +// depends on Backend; unused fields stay zero. +type Context struct { + Backend string `toml:"backend"` + + // k8s + Namespace string `toml:"namespace"` + Release string `toml:"release"` + KubeContext string `toml:"kube_context"` + NodeSelector map[string]string `toml:"node_selector"` + Tolerations []Toleration `toml:"tolerations"` + Resources *Resources `toml:"resources"` + + // ssh + Host string `toml:"host"` + + // direct + CDPURL string `toml:"cdp_url"` + VNCURL string `toml:"vnc_url"` + + // applied at browser startup regardless of backend + Proxy string `toml:"proxy"` +} + +// Toleration mirrors a Kubernetes toleration passed through to the Helm chart. +type Toleration struct { + Key string `toml:"key"` + Operator string `toml:"operator"` + Value string `toml:"value"` + Effect string `toml:"effect"` +} + +// Resources mirrors a Kubernetes resource requests/limits block. +type Resources struct { + Requests map[string]string `toml:"requests"` + Limits map[string]string `toml:"limits"` +} + +// Profile is a named cuttle seed with a storage policy. +type Profile struct { + Storage string `toml:"storage"` // "local" (default) | "remote" +} + +var errUnknownContext = errors.New("unknown context") + +// Load reads the config from the default XDG path. A missing file is not an +// error: it returns a Config carrying only the built-in local context. +func Load() (*Config, error) { + return LoadFrom(DefaultPath()) +} + +// DefaultPath is $XDG_CONFIG_HOME/cuttle/config.toml, falling back to +// ~/.config/cuttle/config.toml. +func DefaultPath() string { + return filepath.Join(xdg.ConfigDir(), "cuttle", "config.toml") +} + +// LoadFrom reads a specific config file. A missing file yields the built-in +// local context. Unknown keys are rejected (strict mode) so a typo surfaces +// instead of silently doing nothing. +func LoadFrom(path string) (*Config, error) { + cfg := &Config{} + data, err := os.ReadFile(path) + switch { + case errors.Is(err, os.ErrNotExist): + // zero config; fall through to built-in local injection + case err != nil: + return nil, fmt.Errorf("reading config %s: %w", path, err) + default: + dec := toml.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(cfg); err != nil { + return nil, fmt.Errorf("parsing config %s: %w", path, err) + } + } + + if cfg.Contexts == nil { + cfg.Contexts = map[string]Context{} + } + if _, ok := cfg.Contexts[BackendLocal]; !ok { + cfg.Contexts[BackendLocal] = Context{Backend: BackendLocal} + } + return cfg, nil +} + +// Active resolves the active context by name. Precedence: flag > env > +// default_context > built-in "local". A named context that does not exist is an +// error (typo protection). +func (c *Config) Active(flag, env string) (string, Context, error) { + name := BackendLocal + switch { + case flag != "": + name = flag + case env != "": + name = env + case c.DefaultContext != "": + name = c.DefaultContext + } + ctx, ok := c.Contexts[name] + if !ok { + return "", Context{}, fmt.Errorf("%w %q (check %s)", errUnknownContext, name, DefaultPath()) + } + if ctx.Backend == "" { + ctx.Backend = BackendLocal + } + return name, ctx, nil +} + +// Names returns the context names in a stable order (local first, then the rest +// sorted) for listing. +func (c *Config) Names() []string { + names := make([]string, 0, len(c.Contexts)) + for name := range c.Contexts { + if name != BackendLocal { + names = append(names, name) + } + } + slices.Sort(names) + return append([]string{BackendLocal}, names...) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..74d54a3 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,154 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +const sampleTOML = ` +default_context = "cluster" + +[context.cluster] +backend = "k8s" +namespace = "browser" +release = "cuttle" +node_selector = { "glim.sh/browser" = "true" } +proxy = "http://user:pass@proxy.example:8080" + +[context.box] +backend = "ssh" +host = "user@box.example" + +[context.tailnet] +backend = "direct" +cdp_url = "http://cuttle.example:9222" +vnc_url = "http://cuttle.example:6080" + +[profile.default] +storage = "local" + +[profile.work] +storage = "remote" +` + +func writeConfig(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "config.toml") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +func TestLoadFromMissingFileYieldsBuiltinLocal(t *testing.T) { + cfg, err := LoadFrom(filepath.Join(t.TempDir(), "does-not-exist.toml")) + if err != nil { + t.Fatalf("missing file should not error: %v", err) + } + name, ctx, err := cfg.Active("", "") + if err != nil { + t.Fatalf("Active: %v", err) + } + if name != BackendLocal || ctx.Backend != BackendLocal { + t.Fatalf("want built-in local, got name=%q backend=%q", name, ctx.Backend) + } +} + +func TestActivePrecedence(t *testing.T) { + cfg, err := LoadFrom(writeConfig(t, sampleTOML)) + if err != nil { + t.Fatalf("load: %v", err) + } + tests := []struct { + name string + flag string + env string + want string + }{ + {"flag wins over env and default", "box", "tailnet", "box"}, + {"env wins over default", "", "box", "box"}, + {"default_context used when no flag/env", "", "", "cluster"}, + {"flag over default", "tailnet", "", "tailnet"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _, err := cfg.Active(tt.flag, tt.env) + if err != nil { + t.Fatalf("Active: %v", err) + } + if got != tt.want { + t.Fatalf("got %q want %q", got, tt.want) + } + }) + } +} + +func TestActiveBuiltinLocalWhenNoDefault(t *testing.T) { + cfg, err := LoadFrom(writeConfig(t, "[context.box]\nbackend = \"ssh\"\nhost = \"h\"\n")) + if err != nil { + t.Fatalf("load: %v", err) + } + got, ctx, err := cfg.Active("", "") + if err != nil { + t.Fatalf("Active: %v", err) + } + if got != BackendLocal || ctx.Backend != BackendLocal { + t.Fatalf("want built-in local, got %q/%q", got, ctx.Backend) + } +} + +func TestActiveUnknownContextErrors(t *testing.T) { + cfg, err := LoadFrom(writeConfig(t, sampleTOML)) + if err != nil { + t.Fatalf("load: %v", err) + } + if _, _, err := cfg.Active("typo", ""); err == nil { + t.Fatal("expected error for unknown context") + } +} + +func TestStrictModeRejectsUnknownKey(t *testing.T) { + _, err := LoadFrom(writeConfig(t, "defaultcontext = \"x\"\n")) // typo: should be default_context + if err == nil { + t.Fatal("expected strict-mode error for unknown key") + } +} + +func TestParsedFields(t *testing.T) { + cfg, err := LoadFrom(writeConfig(t, sampleTOML)) + if err != nil { + t.Fatalf("load: %v", err) + } + cluster := cfg.Contexts["cluster"] + if cluster.Namespace != "browser" || cluster.Release != "cuttle" { + t.Fatalf("cluster fields: %+v", cluster) + } + if cluster.NodeSelector["glim.sh/browser"] != "true" { + t.Fatalf("node_selector not parsed: %+v", cluster.NodeSelector) + } + if cfg.Contexts["tailnet"].CDPURL != "http://cuttle.example:9222" { + t.Fatalf("direct cdp_url: %q", cfg.Contexts["tailnet"].CDPURL) + } + if cfg.Profiles["work"].Storage != StorageRemote { + t.Fatalf("profile storage: %+v", cfg.Profiles["work"]) + } +} + +func TestNamesLocalFirstThenSorted(t *testing.T) { + cfg, err := LoadFrom(writeConfig(t, sampleTOML)) + if err != nil { + t.Fatalf("load: %v", err) + } + got := cfg.Names() + want := []string{"local", "box", "cluster", "tailnet"} + if len(got) != len(want) { + t.Fatalf("got %v want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v want %v", got, want) + } + } +} diff --git a/internal/fingerprint/args.go b/internal/fingerprint/args.go new file mode 100644 index 0000000..b8953f6 --- /dev/null +++ b/internal/fingerprint/args.go @@ -0,0 +1,196 @@ +// Package fingerprint builds the stealth Chrome argument vector and resolves +// proxy geo/exit-IP metadata. Its output is pinned by the golden snapshot in +// testdata and regression-tested, because a silent drift is a silent stealth +// loss. +package fingerprint + +import ( + "fmt" + "math/rand/v2" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// ReservedSeed is the sentinel seed that maps to the shared default Chrome +// instance; it is not a valid user-supplied seed. +const ReservedSeed = "__default__" + +var seedRE = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) + +// ValidSeed reports whether name is a legal fingerprint seed: 1-128 chars of +// [A-Za-z0-9_-] and not the reserved default sentinel. Seeds and profile names +// share this grammar, so both the serve multiplexer and local profiles call it. +func ValidSeed(name string) bool { + return name != ReservedSeed && seedRE.MatchString(name) +} + +// systemName and seedSource are overridable so parity tests can pin the platform +// and fingerprint seed the original used. Defaults mirror the runtime. +var ( + systemName = defaultSystemName + seedSource = defaultSeed +) + +func defaultSystemName() string { + switch runtime.GOOS { + case "darwin": + return "Darwin" + case "windows": + return "Windows" + default: + return "Linux" + } +} + +func defaultSeed() int { + // A fingerprint seed, not a security token; math/rand mirrors the original. + return rand.IntN(90000) + 10000 //nolint:gosec // non-cryptographic seed +} + +func getDefaultStealthArgs() []string { + base := []string{"--no-sandbox", fmt.Sprintf("--fingerprint=%d", seedSource())} + if systemName() == "Darwin" { + return append(base, "--fingerprint-platform=macos") + } + return append(base, "--fingerprint-platform=windows") +} + +// BuildArgsInput holds the parameters of the vendored build_args function. +type BuildArgsInput struct { + StealthArgs bool + ExtraArgs []string + Timezone string + Locale string + Headless bool + ExtensionPaths []string + StartMaximized bool +} + +// BuildArgs combines stealth defaults, user args, and locale/timezone flags, +// deduplicating by flag key (everything before '='). Priority: stealth defaults +// < user args < dedicated params. Insertion order is preserved, and updating an +// existing key keeps its original position. +func BuildArgs(in BuildArgsInput) []string { + seen := newOrderedArgs() + + if in.StealthArgs { + for _, arg := range getDefaultStealthArgs() { + seen.set(argKey(arg), arg) + } + } + + if !in.Headless || systemName() == "Windows" { + seen.set("--ignore-gpu-blocklist", "--ignore-gpu-blocklist") + } + + for _, arg := range in.ExtraArgs { + seen.set(argKey(arg), arg) + } + + if in.Timezone != "" { + seen.set("--fingerprint-timezone", "--fingerprint-timezone="+in.Timezone) + } + if in.Locale != "" { + for _, key := range []string{"--lang", "--fingerprint-locale"} { + seen.set(key, key+"="+in.Locale) + } + } + + if len(in.ExtensionPaths) > 0 { + absPaths := make([]string, len(in.ExtensionPaths)) + for i, p := range in.ExtensionPaths { + ap, err := filepath.Abs(p) + if err != nil { + ap = p + } + absPaths[i] = ap + } + extVal := strings.Join(absPaths, ",") + seen.set("--load-extension", "--load-extension="+extVal) + seen.set("--disable-extensions-except", "--disable-extensions-except="+extVal) + } + + if in.StartMaximized && !seen.has("--start-maximized") && + !seen.has("--window-size") && !seen.has("--window-position") { + seen.set("--start-maximized", "--start-maximized") + } + + return seen.values() +} + +func argKey(arg string) string { + key, _, _ := strings.Cut(arg, "=") + return key +} + +// ForkParityArgs replicates clark/clearcote's own launcher flag set, which the +// vendored build_args (tuned for the Pro binary) omits but the fork binaries +// require: an explicit --user-agent matching navigator.userAgent, the ungoogled +// canvas/client-rects noise switches, UA-CH brand/platform coherence, a Windows +// font dir, the Accept-Language header, and a residential network profile. +// Returns nil unless a fork binary is selected via CLOAKBROWSER_BINARY_PATH. +func ForkParityArgs(locale, proxy string) []string { + if os.Getenv(BinaryPathEnv) == "" { + return nil + } + lang := locale + if lang == "" { + lang = "en-US" + } + base, _, _ := strings.Cut(lang, "-") + acceptLang := "--accept-lang=" + lang + if base != lang { + acceptLang = "--accept-lang=" + lang + "," + base + } + args := []string{ + "--fingerprint-platform=windows", + "--fingerprint-platform-version=19.0.0", + "--fingerprint-brand=Chrome", + "--fingerprint-brand-version=148.0.0.0", + "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", + "--fingerprint-fonts-dir=/opt/winfonts", + "--fingerprinting-client-rects-noise", + "--fingerprinting-canvas-measuretext-noise", + "--fingerprinting-canvas-image-data-noise", + acceptLang, + } + if proxy != "" { + args = append(args, "--fingerprint-network-profile=residential") + } + return args +} + +// orderedArgs is an insertion-ordered string map that mirrors CPython dict +// semantics: a repeated key updates its value in place without moving. +type orderedArgs struct { + keys []string + vals map[string]string +} + +func newOrderedArgs() *orderedArgs { + return &orderedArgs{vals: make(map[string]string)} +} + +func (o *orderedArgs) set(key, val string) { + if _, ok := o.vals[key]; !ok { + o.keys = append(o.keys, key) + } + o.vals[key] = val +} + +func (o *orderedArgs) has(key string) bool { + _, ok := o.vals[key] + return ok +} + +func (o *orderedArgs) values() []string { + out := make([]string, len(o.keys)) + for i, k := range o.keys { + out[i] = o.vals[k] + } + return out +} diff --git a/internal/fingerprint/binary.go b/internal/fingerprint/binary.go new file mode 100644 index 0000000..9ce50a8 --- /dev/null +++ b/internal/fingerprint/binary.go @@ -0,0 +1,32 @@ +package fingerprint + +import ( + "errors" + "fmt" + "io/fs" + "os" +) + +// BinaryPathEnv selects the stealth Chromium fork binary. +const BinaryPathEnv = "CLOAKBROWSER_BINARY_PATH" + +var ( + errBinaryPathUnset = errors.New(BinaryPathEnv + " is not set; cuttle ships no binary download, point it at a local stealth Chromium build") + errBinaryPathMissing = errors.New(BinaryPathEnv + " points at a file that does not exist") +) + +// EnsureBinary resolves the stealth Chromium binary from CLOAKBROWSER_BINARY_PATH, +// erroring clearly when the variable is unset or points at a missing file. +func EnsureBinary() (string, error) { + path := os.Getenv(BinaryPathEnv) + if path == "" { + return "", errBinaryPathUnset + } + if _, err := os.Stat(path); err != nil { //nolint:gosec // operator-supplied binary path by design + if errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("%w: %q", errBinaryPathMissing, path) + } + return "", fmt.Errorf("stat %s: %w", BinaryPathEnv, err) + } + return path, nil +} diff --git a/internal/fingerprint/binary_test.go b/internal/fingerprint/binary_test.go new file mode 100644 index 0000000..68e0d93 --- /dev/null +++ b/internal/fingerprint/binary_test.go @@ -0,0 +1,44 @@ +package fingerprint + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestEnsureBinary(t *testing.T) { + realBin := filepath.Join(t.TempDir(), "chrome") + if err := os.WriteFile(realBin, []byte("#!/bin/sh\n"), 0o600); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + value string + want string + wantErr error + }{ + {name: "unset", value: "", wantErr: errBinaryPathUnset}, + {name: "missing", value: "/no/such/chrome", wantErr: errBinaryPathMissing}, + {name: "present", value: realBin, want: realBin}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(BinaryPathEnv, tt.value) + got, err := EnsureBinary() + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/fingerprint/geoip.go b/internal/fingerprint/geoip.go new file mode 100644 index 0000000..b2d6b0d --- /dev/null +++ b/internal/fingerprint/geoip.go @@ -0,0 +1,287 @@ +package fingerprint + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + geoip2 "github.com/oschwald/geoip2-golang/v2" + "golang.org/x/net/proxy" +) + +// CountryLocaleMap maps an ISO 3166-1 alpha-2 country code to a BCP 47 locale. +// Pinned by the golden snapshot; a change must be a reviewed golden diff. +var CountryLocaleMap = map[string]string{ + "US": "en-US", "GB": "en-GB", "AU": "en-AU", "CA": "en-CA", "NZ": "en-NZ", + "IE": "en-IE", "ZA": "en-ZA", "SG": "en-SG", + "DE": "de-DE", "AT": "de-AT", "CH": "de-CH", + "FR": "fr-FR", "BE": "fr-BE", + "ES": "es-ES", "MX": "es-MX", "AR": "es-AR", "CO": "es-CO", "CL": "es-CL", + "BR": "pt-BR", "PT": "pt-PT", + "IT": "it-IT", "NL": "nl-NL", + "JP": "ja-JP", "KR": "ko-KR", "CN": "zh-CN", "TW": "zh-TW", "HK": "zh-HK", + "RU": "ru-RU", "UA": "uk-UA", "PL": "pl-PL", "CZ": "cs-CZ", "RO": "ro-RO", + "IL": "he-IL", "TR": "tr-TR", "SA": "ar-SA", "AE": "ar-AE", "EG": "ar-EG", + "IN": "hi-IN", "ID": "id-ID", "PH": "en-PH", + "TH": "th-TH", "VN": "vi-VN", "MY": "ms-MY", + "SE": "sv-SE", "NO": "nb-NO", "DK": "da-DK", "FI": "fi-FI", + "GR": "el-GR", "HU": "hu-HU", "BG": "bg-BG", + "SI": "sl-SI", "SK": "sk-SK", "HR": "hr-HR", "RS": "sr-RS", "LT": "lt-LT", + "LV": "lv-LV", "EE": "et-EE", "IS": "is-IS", "LU": "fr-LU", "MT": "en-MT", + "CY": "el-CY", "MD": "ro-MD", "BY": "ru-BY", "GE": "ka-GE", "AL": "sq-AL", + "MK": "mk-MK", "BA": "bs-BA", + "PE": "es-PE", "VE": "es-VE", "EC": "es-EC", "UY": "es-UY", "CR": "es-CR", + "DO": "es-DO", "GT": "es-GT", "BO": "es-BO", "PY": "es-PY", + "PK": "en-PK", "BD": "bn-BD", "LK": "si-LK", "KZ": "ru-KZ", "IR": "fa-IR", + "IQ": "ar-IQ", "JO": "ar-JO", "LB": "ar-LB", "KW": "ar-KW", "QA": "ar-QA", + "OM": "ar-OM", "BH": "ar-BH", + "NG": "en-NG", "KE": "en-KE", "MA": "fr-MA", "DZ": "ar-DZ", "TN": "ar-TN", + "GH": "en-GH", + "AM": "hy-AM", "AZ": "az-AZ", "UZ": "uz-UZ", "KG": "ky-KG", "TJ": "tg-TJ", + "TM": "tk-TM", + "ME": "sr-ME", "XK": "sq-XK", "LI": "de-LI", "MC": "fr-MC", "AD": "ca-AD", + "MM": "my-MM", "KH": "km-KH", "LA": "lo-LA", "MN": "mn-MN", "BN": "ms-BN", + "MO": "zh-MO", + "YE": "ar-YE", "SY": "ar-SY", "PS": "ar-PS", "LY": "ar-LY", + "ET": "am-ET", "TZ": "sw-TZ", "UG": "en-UG", "SN": "fr-SN", "CI": "fr-CI", + "CM": "fr-CM", "AO": "pt-AO", "MZ": "pt-MZ", "ZM": "en-ZM", "ZW": "en-ZW", + "HN": "es-HN", "NI": "es-NI", "SV": "es-SV", "PA": "es-PA", "JM": "en-JM", + "TT": "en-TT", "PR": "es-PR", +} + +const ( + geoIPDBURL = "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb" + geoIPDBFilename = "GeoLite2-City.mmdb" + geoIPUpdateInterval = 30 * 24 * time.Hour + echoTimeout = 10 * time.Second +) + +var ( + errNoExitIP = errors.New("failed to discover exit IP") + errGeoIPDownStatus = errors.New("geoip download: unexpected status") +) + +// ipEchoURLs are public IP-echo services queried (through the proxy when set) to +// discover the egress IP. +var ipEchoURLs = []string{ + "https://api.ipify.org", + "https://checkip.amazonaws.com", + "https://ifconfig.me/ip", +} + +// ExitIPFunc resolves the egress IP for a proxy URL ("" = the machine's own +// public IP). Injected so callers can stub network access in tests. +type ExitIPFunc func(proxyURL string) (string, error) + +// GeoResolver resolves timezone/locale/exit-IP from a proxy. All fields are +// injectable for hermetic testing; the zero value is not usable - construct via +// [NewGeoResolver]. +type GeoResolver struct { + ExitIP ExitIPFunc + DBPath func() string + // ResolveHost DNS-resolves the proxy's own hostname to an IP. It is the + // fallback egress IP when every echo service is unreachable but the proxy + // host still resolves (common for datacenter proxies that block outbound to + // the echo endpoints). + ResolveHost func(proxyURL string) string +} + +// NewGeoResolver returns a resolver wired to the real echo services, the cached +// mmdb (downloaded on first use), and DNS-based proxy-host fallback. +func NewGeoResolver() GeoResolver { + return GeoResolver{ExitIP: DefaultExitIP, DBPath: ensureGeoIPDB, ResolveHost: resolveProxyHostIP} +} + +// ResolveProxyGeoWithIP returns (timezone, locale, exitIP). When the echo +// services fail, it falls back to DNS-resolving the proxy hostname (gateway geo) +// rather than dropping the IP, so WebRTC never leaks the real address behind a +// proxy. A missing or failed mmdb still returns the exit IP; any lookup failure +// degrades gracefully rather than erroring. +func (r GeoResolver) ResolveProxyGeoWithIP(proxyURL string) (string, string, string) { + ip, err := r.ExitIP(proxyURL) + if (err != nil || ip == "") && proxyURL != "" && r.ResolveHost != nil { + ip = r.ResolveHost(proxyURL) + } + if ip == "" { + return "", "", "" + } + dbPath := "" + if r.DBPath != nil { + dbPath = r.DBPath() + } + if dbPath == "" { + return "", "", ip + } + db, err := geoip2.Open(dbPath) + if err != nil { + return "", "", ip + } + defer func() { _ = db.Close() }() + + addr, err := netip.ParseAddr(ip) + if err != nil { + return "", "", ip + } + city, err := db.City(addr) + if err != nil { + return "", "", ip + } + loc := "" + if city.Country.ISOCode != "" { + loc = CountryLocaleMap[city.Country.ISOCode] + } + return city.Location.TimeZone, loc, ip +} + +// DefaultExitIP discovers the egress IP by querying the echo services through +// the proxy (or directly when proxyURL is empty). +func DefaultExitIP(proxyURL string) (string, error) { + client, err := echoClient(proxyURL) + if err != nil { + return "", err + } + for _, u := range ipEchoURLs { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, u, nil) + if err != nil { + continue + } + resp, err := client.Do(req) + if err != nil { + continue + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 64)) + _ = resp.Body.Close() + if err != nil { + continue + } + ip := strings.TrimSpace(string(body)) + if _, err := netip.ParseAddr(ip); err != nil { + continue + } + return ip, nil + } + return "", errNoExitIP +} + +// resolveProxyHostIP extracts the proxy hostname and resolves it to an IP: a +// literal IP is returned as-is, otherwise the first DNS result. Returns "" when +// the host is absent or unresolvable. +func resolveProxyHostIP(proxyURL string) string { + host := urlsplit(proxyURL).hostname() + if host == "" { + return "" + } + if addr, err := netip.ParseAddr(host); err == nil { + return addr.String() + } + ips, err := net.DefaultResolver.LookupIPAddr(context.Background(), host) + if err != nil || len(ips) == 0 { + return "" + } + return ips[0].IP.String() +} + +func echoClient(proxyURL string) (*http.Client, error) { + if proxyURL == "" { + return &http.Client{Timeout: echoTimeout}, nil + } + u, err := url.Parse(proxyURL) + if err != nil { + return nil, err //nolint:wrapcheck + } + switch strings.ToLower(u.Scheme) { + case "socks5", "socks5h": + var auth *proxy.Auth + if u.User != nil { + pw, _ := u.User.Password() + auth = &proxy.Auth{User: u.User.Username(), Password: pw} + } + dialer, err := proxy.SOCKS5("tcp", u.Host, auth, proxy.Direct) + if err != nil { + return nil, err //nolint:wrapcheck + } + tr := &http.Transport{} + if cd, ok := dialer.(proxy.ContextDialer); ok { + tr.DialContext = cd.DialContext + } + return &http.Client{Transport: tr, Timeout: echoTimeout}, nil + default: + return &http.Client{ + Transport: &http.Transport{Proxy: http.ProxyURL(u)}, + Timeout: echoTimeout, + }, nil + } +} + +func geoIPDir() (string, error) { + base, err := os.UserCacheDir() + if err != nil { + return "", err //nolint:wrapcheck + } + return filepath.Join(base, "cuttle", "geoip"), nil +} + +// ensureGeoIPDB returns the cached mmdb path, downloading it on first use and +// triggering a background refresh once older than the update interval. Returns +// "" when unavailable so geo resolution degrades to exit-IP-only. +func ensureGeoIPDB() string { + dir, err := geoIPDir() + if err != nil { + return "" + } + dbPath := filepath.Join(dir, geoIPDBFilename) + if info, err := os.Stat(dbPath); err == nil { + if time.Since(info.ModTime()) >= geoIPUpdateInterval { + go func() { _ = downloadGeoIPDB(dbPath) }() + } + return dbPath + } + if err := downloadGeoIPDB(dbPath); err != nil { + return "" + } + return dbPath +} + +func downloadGeoIPDB(dest string) error { + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return err //nolint:wrapcheck + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, geoIPDBURL, nil) + if err != nil { + return err //nolint:wrapcheck + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err //nolint:wrapcheck + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%w: %s", errGeoIPDownStatus, resp.Status) + } + tmp, err := os.CreateTemp(filepath.Dir(dest), "*.tmp") + if err != nil { + return err //nolint:wrapcheck + } + tmpName := tmp.Name() + if _, err := io.Copy(tmp, resp.Body); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err //nolint:wrapcheck + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err //nolint:wrapcheck + } + return os.Rename(tmpName, dest) //nolint:wrapcheck +} diff --git a/internal/fingerprint/geoip_test.go b/internal/fingerprint/geoip_test.go new file mode 100644 index 0000000..d2bd306 --- /dev/null +++ b/internal/fingerprint/geoip_test.go @@ -0,0 +1,56 @@ +package fingerprint + +import ( + "errors" + "testing" +) + +const testExitIP = "203.0.113.7" + +var errNoRoute = errors.New("no route") + +func TestResolveProxyGeoWithIPDegrades(t *testing.T) { + tests := []struct { + name string + exitIP ExitIPFunc + dbPath func() string + resolveHost func(string) string + wantTZ string + wantLocale string + wantIP string + }{ + { + name: "echo and host resolution both fail yields nothing", + exitIP: func(string) (string, error) { return "", errNoRoute }, + resolveHost: func(string) string { return "" }, + }, + { + name: "echo failure falls back to proxy host resolution", + exitIP: func(string) (string, error) { return "", errNoRoute }, + resolveHost: func(string) string { return testExitIP }, + dbPath: func() string { return "" }, + wantIP: testExitIP, + }, + { + name: "no db degrades to exit-ip only", + exitIP: func(string) (string, error) { return testExitIP, nil }, + dbPath: func() string { return "" }, + wantIP: testExitIP, + }, + { + name: "missing db file degrades to exit-ip only", + exitIP: func(string) (string, error) { return testExitIP, nil }, + dbPath: func() string { return "testdata/does-not-exist.mmdb" }, + wantIP: testExitIP, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := GeoResolver{ExitIP: tt.exitIP, DBPath: tt.dbPath, ResolveHost: tt.resolveHost} + tz, locale, ip := r.ResolveProxyGeoWithIP("http://proxy.example:8080") + if tz != tt.wantTZ || locale != tt.wantLocale || ip != tt.wantIP { + t.Errorf("got (%q,%q,%q), want (%q,%q,%q)", tz, locale, ip, tt.wantTZ, tt.wantLocale, tt.wantIP) + } + }) + } +} diff --git a/internal/fingerprint/golden_update_test.go b/internal/fingerprint/golden_update_test.go new file mode 100644 index 0000000..b752cad --- /dev/null +++ b/internal/fingerprint/golden_update_test.go @@ -0,0 +1,374 @@ +package fingerprint + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "slices" + "testing" +) + +var update = flag.Bool("update", false, "regenerate testdata/golden.json") + +const exitIPStub = "203.0.113.7" + +// countryLocaleKeyOrder is the emission order of country_locale_map. Go maps +// serialize with sorted keys, but the golden preserves the original insertion +// order (which matches the CountryLocaleMap literal), so the order is pinned +// here and the values are read from the live map. +var countryLocaleKeyOrder = []string{ + "US", "GB", "AU", "CA", "NZ", "IE", "ZA", "SG", "DE", "AT", + "CH", "FR", "BE", "ES", "MX", "AR", "CO", "CL", "BR", "PT", + "IT", "NL", "JP", "KR", "CN", "TW", "HK", "RU", "UA", "PL", + "CZ", "RO", "IL", "TR", "SA", "AE", "EG", "IN", "ID", "PH", + "TH", "VN", "MY", "SE", "NO", "DK", "FI", "GR", "HU", "BG", + "SI", "SK", "HR", "RS", "LT", "LV", "EE", "IS", "LU", "MT", + "CY", "MD", "BY", "GE", "AL", "MK", "BA", "PE", "VE", "EC", + "UY", "CR", "DO", "GT", "BO", "PY", "PK", "BD", "LK", "KZ", + "IR", "IQ", "JO", "LB", "KW", "QA", "OM", "BH", "NG", "KE", + "MA", "DZ", "TN", "GH", "AM", "AZ", "UZ", "KG", "TJ", "TM", + "ME", "XK", "LI", "MC", "AD", "MM", "KH", "LA", "MN", "BN", + "MO", "YE", "SY", "PS", "LY", "ET", "TZ", "UG", "SN", "CI", + "CM", "AO", "MZ", "ZM", "ZW", "HN", "NI", "SV", "PA", "JM", + "TT", "PR", +} + +// localeMapDump serializes CountryLocaleMap in countryLocaleKeyOrder. The outer +// json.Encoder re-indents this compact object into the surrounding 2-space tree. +type localeMapDump struct{} + +func (localeMapDump) MarshalJSON() ([]byte, error) { + var b bytes.Buffer + b.WriteByte('{') + for i, k := range countryLocaleKeyOrder { + if i > 0 { + b.WriteByte(',') + } + kb, err := json.Marshal(k) + if err != nil { + return nil, err + } + vb, err := json.Marshal(CountryLocaleMap[k]) + if err != nil { + return nil, err + } + b.Write(kb) + b.WriteByte(':') + b.Write(vb) + } + b.WriteByte('}') + return b.Bytes(), nil +} + +type ioCaseDump struct { + Input string `json:"input"` + Output string `json:"output"` +} + +type stealthCaseDump struct { + System string `json:"system"` + Seed int `json:"seed"` + Output []string `json:"output"` +} + +type splitCaseDump struct { + Input string `json:"input"` + Server string `json:"server"` + Username string `json:"username"` + Password string `json:"password"` +} + +type forkCaseDump struct { + Locale string `json:"locale"` + Proxy *string `json:"proxy"` + Output []string `json:"output"` +} + +type webrtcCaseDump struct { + InputArgs []string `json:"input_args"` + Proxy *string `json:"proxy"` + ExitIP *string `json:"exit_ip"` + Output []string `json:"output"` +} + +// buildInputDump mirrors the vendored build_args kwargs. Field order and the +// omitempty set reproduce the original's per-case presence exactly: stealth_args +// and extra_args are always emitted (extra_args as null when unset); the rest +// appear only when supplied. +type buildInputDump struct { + StealthArgs bool `json:"stealth_args"` + ExtraArgs []string `json:"extra_args"` + Timezone string `json:"timezone,omitempty"` + Locale string `json:"locale,omitempty"` + Headless *bool `json:"headless,omitempty"` + ExtensionPaths []string `json:"extension_paths,omitempty"` + StartMaximized *bool `json:"start_maximized,omitempty"` +} + +type buildCaseDump struct { + Name string `json:"name"` + Input buildInputDump `json:"input"` + Output []string `json:"output"` +} + +type composeCaseDump struct { + Seed *string `json:"seed"` + Proxy *string `json:"proxy"` + Timezone *string `json:"timezone"` + Locale *string `json:"locale"` + Webrtc string `json:"webrtc"` + Output []string `json:"output"` +} + +// goldenDump fixes the top-level key order of golden.json. It intentionally +// differs from goldenFile (the reader), whose field order is irrelevant to +// unmarshaling. +type goldenDump struct { + ExitIPStub string `json:"exit_ip_stub"` + CountryLocaleMap localeMapDump `json:"country_locale_map"` + DefaultStealthArgs []stealthCaseDump `json:"default_stealth_args"` + EnsureProxyScheme []ioCaseDump `json:"ensure_proxy_scheme"` + NormalizeSocks []ioCaseDump `json:"normalize_socks"` + SplitProxyAuth []splitCaseDump `json:"split_proxy_auth"` + ForkParityArgs []forkCaseDump `json:"fork_parity_args"` + ResolveWebrtc []webrtcCaseDump `json:"resolve_webrtc"` + BuildArgs []buildCaseDump `json:"build_args"` + ComposeArgv []composeCaseDump `json:"compose_argv"` +} + +// TestGolden regenerates testdata/golden.json from the Go primitives when run +// with -update, and otherwise asserts the committed file still matches what the +// generator produces (a regression snapshot of the ported primitives). +func TestGolden(t *testing.T) { + data := buildGolden(t) + + if *update { + if err := os.WriteFile("testdata/golden.json", data, 0o644); err != nil { + t.Fatalf("write golden: %v", err) + } + return + } + + committed, err := os.ReadFile("testdata/golden.json") + if err != nil { + t.Fatalf("read golden: %v", err) + } + if !bytes.Equal(data, committed) { + t.Errorf("golden.json is stale; run `go test ./internal/fingerprint -run TestGolden -update`") + } +} + +func buildGolden(t *testing.T) []byte { + t.Helper() + t.Setenv(BinaryPathEnv, "/opt/clark/chrome") + + dump := goldenDump{ + ExitIPStub: exitIPStub, + DefaultStealthArgs: dumpDefaultStealthArgs(), + EnsureProxyScheme: dumpEnsureProxyScheme(), + NormalizeSocks: dumpNormalizeSocks(), + SplitProxyAuth: dumpSplitProxyAuth(), + ForkParityArgs: dumpForkParityArgs(), + ResolveWebrtc: dumpResolveWebrtc(), + } + + pinLinux(t) + dump.BuildArgs = dumpBuildArgs() + dump.ComposeArgv = dumpComposeArgv() + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(dump); err != nil { + t.Fatalf("encode golden: %v", err) + } + return buf.Bytes() +} + +func dumpDefaultStealthArgs() []stealthCaseDump { + systems := []string{"Linux", "Darwin", "Windows"} + out := make([]stealthCaseDump, len(systems)) + for i, sys := range systems { + out[i] = stealthCaseDump{System: sys, Seed: pinnedSeed, Output: stealthArgsFor(sys, pinnedSeed)} + } + return out +} + +func stealthArgsFor(system string, seed int) []string { + origSystem, origSeed := systemName, seedSource + defer func() { systemName, seedSource = origSystem, origSeed }() + systemName = func() string { return system } + seedSource = func() int { return seed } + return getDefaultStealthArgs() +} + +func dumpEnsureProxyScheme() []ioCaseDump { + inputs := []string{ + "proxy.example:8080", + "http://proxy.example:8080", + "socks5://proxy.example:1080", + } + out := make([]ioCaseDump, len(inputs)) + for i, in := range inputs { + out[i] = ioCaseDump{Input: in, Output: EnsureProxyScheme(in)} + } + return out +} + +func dumpNormalizeSocks() []ioCaseDump { + inputs := []string{ + "http://proxy.example:8080", + "socks5://proxy.example:1080", + "http://user:p%40ss@proxy.example:8080", + "socks5://user:p%40ss@proxy.example:1080", + "socks5://user:p@ss=word@proxy.example:1080", + "socks5://USER:P@SS@HOST.example:1080", + "socks5://user:@proxy.example:1080", + "socks5://user@proxy.example:1080", + "socks5://user:pass@[2001:db8::1]:1080", + "socks5://proxy.example:not-a-port", + } + out := make([]ioCaseDump, len(inputs)) + for i, in := range inputs { + out[i] = ioCaseDump{Input: in, Output: NormalizeSocksStringURL(in)} + } + return out +} + +func dumpSplitProxyAuth() []splitCaseDump { + inputs := []string{ + "http://bob:secret@proxy.example:8080", + "http://bob:secret@Proxy.EXAMPLE:8080", + "http://user%40host:p%40ss@host.example:8080", + "https://u:p@[2001:db8::1]:8443", + "http://bob@proxy.example:8080", + "http://user:@proxy.example:8080", + "http://proxy.example:8080", + "socks5://user:pass@proxy.example:1080", + } + out := make([]splitCaseDump, len(inputs)) + for i, in := range inputs { + server, user, pass := SplitProxyAuth(in) + out[i] = splitCaseDump{Input: in, Server: server, Username: user, Password: pass} + } + return out +} + +func dumpForkParityArgs() []forkCaseDump { + cases := []struct { + locale string + proxy *string + }{ + {"", nil}, + {"en-US", nil}, + {"de-DE", new("http://p:1")}, + {"fr", new("socks5://p:1")}, + {"", new("http://p:1")}, + } + out := make([]forkCaseDump, len(cases)) + for i, c := range cases { + out[i] = forkCaseDump{Locale: c.locale, Proxy: c.proxy, Output: ForkParityArgs(c.locale, deref(c.proxy))} + } + return out +} + +func dumpResolveWebrtc() []webrtcCaseDump { + cases := []struct { + args []string + proxy *string + exitIP *string + }{ + {[]string{"--fingerprint=1", "--no-sandbox"}, new("http://proxy.example:8080"), new(exitIPStub)}, + {[]string{"--fingerprint-webrtc-ip=auto"}, new("http://proxy.example:8080"), new(exitIPStub)}, + {[]string{"--fingerprint-webrtc-ip=auto"}, new("socks5://proxy.example:1080"), new(exitIPStub)}, + {[]string{"--fingerprint-webrtc-ip=auto"}, nil, new(exitIPStub)}, + {[]string{"--x", "--fingerprint-webrtc-ip=auto"}, new("http://proxy.example:8080"), nil}, + } + out := make([]webrtcCaseDump, len(cases)) + for i, c := range cases { + exitIP := deref(c.exitIP) + got := ResolveWebRTCArgs(slices.Clone(c.args), deref(c.proxy), func(string) string { return exitIP }) + out[i] = webrtcCaseDump{InputArgs: c.args, Proxy: c.proxy, ExitIP: c.exitIP, Output: got} + } + return out +} + +func dumpBuildArgs() []buildCaseDump { + inputs := []struct { + name string + input buildInputDump + }{ + {"headed-adds-gpu-flag", buildInputDump{StealthArgs: true, ExtraArgs: []string{"--fingerprint=1"}, Headless: new(false)}}, + {"timezone-only", buildInputDump{StealthArgs: true, Timezone: "Europe/Berlin"}}, + {"locale-only", buildInputDump{StealthArgs: true, Locale: "de-DE"}}, + {"no-stealth", buildInputDump{StealthArgs: false, ExtraArgs: []string{"--foo=bar"}, Timezone: "UTC", Locale: "en-US"}}, + {"override-fingerprint", buildInputDump{StealthArgs: true, ExtraArgs: []string{"--fingerprint=99999", "--proxy-server=http://h:1"}}}, + {"extensions", buildInputDump{StealthArgs: true, ExtraArgs: []string{"--fingerprint=1"}, ExtensionPaths: []string{"/opt/ext/a", "/opt/ext/b"}}}, + {"start-maximized", buildInputDump{StealthArgs: true, ExtraArgs: []string{"--fingerprint=1"}, StartMaximized: new(true)}}, + {"start-maximized-suppressed", buildInputDump{StealthArgs: true, ExtraArgs: []string{"--fingerprint=1", "--window-size=800,600"}, StartMaximized: new(true)}}, + } + out := make([]buildCaseDump, len(inputs)) + for i, c := range inputs { + out[i] = buildCaseDump{Name: c.name, Input: c.input, Output: BuildArgs(buildArgsInputFrom(c.input))} + } + return out +} + +// buildArgsInputFrom mirrors TestBuildArgsParity: Headless defaults to true. +func buildArgsInputFrom(d buildInputDump) BuildArgsInput { + in := BuildArgsInput{ + StealthArgs: d.StealthArgs, + ExtraArgs: d.ExtraArgs, + Timezone: d.Timezone, + Locale: d.Locale, + ExtensionPaths: d.ExtensionPaths, + Headless: true, + } + if d.Headless != nil { + in.Headless = *d.Headless + } + if d.StartMaximized != nil { + in.StartMaximized = *d.StartMaximized + } + return in +} + +func dumpComposeArgv() []composeCaseDump { + seeds := []*string{nil, new("12345")} + proxies := []*string{ + nil, + new("http://proxy.example:8080"), + new("socks5://proxy.example:1080"), + new("http://user:p%40ss@proxy.example:8080"), + new("socks5://user:p%40ss@proxy.example:1080"), + new("socks5://user:p@ss=word@proxy.example:1080"), + } + tzLocales := []struct{ tz, locale *string }{ + {nil, nil}, + {new("America/New_York"), new("en-US")}, + } + webrtcs := []string{"none", "auto"} + stub := func(string) string { return exitIPStub } + + out := make([]composeCaseDump, 0, len(seeds)*len(proxies)*len(tzLocales)*len(webrtcs)) + for _, seed := range seeds { + for _, proxy := range proxies { + for _, tl := range tzLocales { + for _, webrtc := range webrtcs { + got := composeArgv(seed, proxy, deref(tl.tz), deref(tl.locale), webrtc, stub) + out = append(out, composeCaseDump{ + Seed: seed, + Proxy: proxy, + Timezone: tl.tz, + Locale: tl.locale, + Webrtc: webrtc, + Output: got, + }) + } + } + } + } + return out +} diff --git a/internal/fingerprint/parity_test.go b/internal/fingerprint/parity_test.go new file mode 100644 index 0000000..78ffe6b --- /dev/null +++ b/internal/fingerprint/parity_test.go @@ -0,0 +1,239 @@ +package fingerprint + +import ( + "encoding/json" + "maps" + "os" + "slices" + "testing" +) + +type goldenFile struct { + ExitIPStub string `json:"exit_ip_stub"` + CountryLocaleMap map[string]string `json:"country_locale_map"` + DefaultStealthArgs []struct { + System string `json:"system"` + Seed int `json:"seed"` + Output []string `json:"output"` + } `json:"default_stealth_args"` + EnsureProxyScheme []struct { + Input string `json:"input"` + Output string `json:"output"` + } `json:"ensure_proxy_scheme"` + NormalizeSocks []struct { + Input string `json:"input"` + Output string `json:"output"` + } `json:"normalize_socks"` + ResolveWebrtc []struct { + InputArgs []string `json:"input_args"` + Proxy *string `json:"proxy"` + ExitIP *string `json:"exit_ip"` + Output []string `json:"output"` + } `json:"resolve_webrtc"` + BuildArgs []struct { + Name string `json:"name"` + Input struct { + StealthArgs *bool `json:"stealth_args"` + ExtraArgs []string `json:"extra_args"` + Timezone string `json:"timezone"` + Locale string `json:"locale"` + Headless *bool `json:"headless"` + ExtensionPaths []string `json:"extension_paths"` + StartMaximized *bool `json:"start_maximized"` + } `json:"input"` + Output []string `json:"output"` + } `json:"build_args"` + ComposeArgv []struct { + Seed *string `json:"seed"` + Proxy *string `json:"proxy"` + Timezone *string `json:"timezone"` + Locale *string `json:"locale"` + Webrtc string `json:"webrtc"` + Output []string `json:"output"` + } `json:"compose_argv"` + SplitProxyAuth []struct { + Input string `json:"input"` + Server string `json:"server"` + Username string `json:"username"` + Password string `json:"password"` + } `json:"split_proxy_auth"` + ForkParityArgs []struct { + Locale string `json:"locale"` + Proxy *string `json:"proxy"` + Output []string `json:"output"` + } `json:"fork_parity_args"` +} + +const pinnedSeed = 55555 + +func loadGolden(t *testing.T) goldenFile { + t.Helper() + raw, err := os.ReadFile("testdata/golden.json") + if err != nil { + t.Fatalf("read golden: %v", err) + } + var g goldenFile + if err := json.Unmarshal(raw, &g); err != nil { + t.Fatalf("unmarshal golden: %v", err) + } + return g +} + +// pinLinux forces the container-target platform and a fixed stealth seed so +// BuildArgs output matches the committed golden snapshot. +func pinLinux(t *testing.T) { + t.Helper() + origSystem, origSeed := systemName, seedSource + systemName = func() string { return "Linux" } + seedSource = func() int { return pinnedSeed } + t.Cleanup(func() { systemName, seedSource = origSystem, origSeed }) +} + +func TestCountryLocaleMapParity(t *testing.T) { + g := loadGolden(t) + if !maps.Equal(CountryLocaleMap, g.CountryLocaleMap) { + t.Errorf("COUNTRY_LOCALE_MAP diverged from golden (%d vs %d entries)", + len(CountryLocaleMap), len(g.CountryLocaleMap)) + } +} + +func TestDefaultStealthArgsParity(t *testing.T) { + g := loadGolden(t) + for _, c := range g.DefaultStealthArgs { + t.Run(c.System, func(t *testing.T) { + origSystem, origSeed := systemName, seedSource + t.Cleanup(func() { systemName, seedSource = origSystem, origSeed }) + systemName = func() string { return c.System } + seedSource = func() int { return c.Seed } + got := getDefaultStealthArgs() + if !slices.Equal(got, c.Output) { + t.Errorf("system %s:\n got %q\nwant %q", c.System, got, c.Output) + } + }) + } +} + +func TestEnsureProxySchemeParity(t *testing.T) { + g := loadGolden(t) + for _, c := range g.EnsureProxyScheme { + if got := EnsureProxyScheme(c.Input); got != c.Output { + t.Errorf("EnsureProxyScheme(%q) = %q, want %q", c.Input, got, c.Output) + } + } +} + +func TestNormalizeSocksParity(t *testing.T) { + g := loadGolden(t) + for _, c := range g.NormalizeSocks { + if got := NormalizeSocksStringURL(c.Input); got != c.Output { + t.Errorf("NormalizeSocksStringURL(%q) = %q, want %q", c.Input, got, c.Output) + } + } +} + +func TestResolveWebRTCArgsParity(t *testing.T) { + g := loadGolden(t) + for _, c := range g.ResolveWebrtc { + proxy := "" + if c.Proxy != nil { + proxy = *c.Proxy + } + exitIP := "" + if c.ExitIP != nil { + exitIP = *c.ExitIP + } + got := ResolveWebRTCArgs(slices.Clone(c.InputArgs), proxy, func(string) string { return exitIP }) + if !slices.Equal(got, c.Output) { + t.Errorf("ResolveWebRTCArgs(%q, %q) = %q, want %q", c.InputArgs, proxy, got, c.Output) + } + } +} + +func TestBuildArgsParity(t *testing.T) { + pinLinux(t) + g := loadGolden(t) + for _, c := range g.BuildArgs { + t.Run(c.Name, func(t *testing.T) { + in := BuildArgsInput{ + ExtraArgs: c.Input.ExtraArgs, + Timezone: c.Input.Timezone, + Locale: c.Input.Locale, + ExtensionPaths: c.Input.ExtensionPaths, + Headless: true, + } + if c.Input.StealthArgs != nil { + in.StealthArgs = *c.Input.StealthArgs + } + if c.Input.Headless != nil { + in.Headless = *c.Input.Headless + } + if c.Input.StartMaximized != nil { + in.StartMaximized = *c.Input.StartMaximized + } + got := BuildArgs(in) + if !slices.Equal(got, c.Output) { + t.Errorf("%s:\n got %q\nwant %q", c.Name, got, c.Output) + } + }) + } +} + +func TestComposeArgvParity(t *testing.T) { + pinLinux(t) + g := loadGolden(t) + stub := func(string) string { return g.ExitIPStub } + for i, c := range g.ComposeArgv { + got := composeArgv(c.Seed, c.Proxy, deref(c.Timezone), deref(c.Locale), c.Webrtc, stub) + if !slices.Equal(got, c.Output) { + t.Errorf("compose[%d] seed=%v proxy=%v tz=%v loc=%v webrtc=%s:\n got %q\nwant %q", + i, c.Seed, c.Proxy, c.Timezone, c.Locale, c.Webrtc, got, c.Output) + } + } +} + +func TestSplitProxyAuthParity(t *testing.T) { + g := loadGolden(t) + for _, c := range g.SplitProxyAuth { + server, user, pass := SplitProxyAuth(c.Input) + if server != c.Server || user != c.Username || pass != c.Password { + t.Errorf("SplitProxyAuth(%q) = (%q,%q,%q), want (%q,%q,%q)", + c.Input, server, user, pass, c.Server, c.Username, c.Password) + } + } +} + +func TestForkParityArgsParity(t *testing.T) { + t.Setenv(BinaryPathEnv, "/opt/clark/chrome") + g := loadGolden(t) + for _, c := range g.ForkParityArgs { + got := ForkParityArgs(c.Locale, deref(c.Proxy)) + if !slices.Equal(got, c.Output) { + t.Errorf("ForkParityArgs(%q, %v) = %q, want %q", c.Locale, c.Proxy, got, c.Output) + } + } +} + +// composeArgv drives proxy + WebRTC through BuildArgs using only the ported +// primitives, so the full argv is exercised for the golden snapshot. +func composeArgv(seed, proxy *string, timezone, locale, webrtc string, exitIP func(string) string) []string { + var extra []string + if seed != nil { + extra = append(extra, "--fingerprint="+*seed) + } + if proxy != nil { + stripped, _, _ := SplitProxyAuth(*proxy) + extra = append(extra, "--proxy-server="+NormalizeSocksStringURL(stripped)) + } + if webrtc == "auto" { + extra = append(extra, "--fingerprint-webrtc-ip=auto") + } + extra = ResolveWebRTCArgs(extra, deref(proxy), exitIP) + return BuildArgs(BuildArgsInput{StealthArgs: true, ExtraArgs: extra, Timezone: timezone, Locale: locale, Headless: true}) +} + +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/internal/fingerprint/proxy.go b/internal/fingerprint/proxy.go new file mode 100644 index 0000000..754ebbe --- /dev/null +++ b/internal/fingerprint/proxy.go @@ -0,0 +1,123 @@ +package fingerprint + +import ( + "slices" + "strconv" + "strings" +) + +// EnsureProxyScheme prepends http:// to a schemeless proxy URL so parsers can +// extract the hostname. +func EnsureProxyScheme(proxyURL string) string { + if strings.Contains(proxyURL, "://") { + return proxyURL + } + return "http://" + proxyURL +} + +func extractProxyURLString(proxy string) string { + if proxy == "" { + return "" + } + return EnsureProxyScheme(proxy) +} + +// NormalizeSocksStringURL re-encodes credentials in a proxy URL so Chromium's +// parser does not truncate them at special characters. It is idempotent on +// already-encoded input and passes unparseable input through unchanged. +func NormalizeSocksStringURL(rawurl string) string { + u := urlparse(rawurl) + if _, _, err := u.port(); err != nil { + return rawurl + } + user, userSet, pass, passSet := u.userinfo() + if !userSet && !passSet { + return rawurl + } + rawUser := "" + if userSet { + rawUser = user + } + encUser := "" + if rawUser != "" { + encUser = pyQuote(pyUnquote(rawUser)) + } + encPass := "" + passPresent := false + if passSet { + passPresent = true + if pass != "" { + encPass = pyQuote(pyUnquote(pass)) + } + } + port, hasPort, _ := u.port() + return assembleProxyURL(u.scheme, u.hostname(), hasPort, port, encUser, encPass, passPresent, u.path, u.params, u.query, u.fragment) +} + +// SplitProxyAuth strips inline credentials from an http(s) proxy URL. Stock +// Chromium and the free forks reject credentials on --proxy-server, so the +// cred-less URL is used there and the username/password are answered over CDP +// (Fetch.continueWithAuth). SOCKS and cred-less proxies pass through unchanged +// with empty credentials. +// +// It byte-matches CPython's urlsplit + SplitResult.username/password (raw, +// percent-encoding preserved) / hostname (lowercased) / port (re-rendered) + +// urlunsplit, so both the argv and the credentials answered to the proxy are +// identical to what CPython's urllib.parse produces. +func SplitProxyAuth(proxy string) (string, string, string) { + u := urlsplit(proxy) + rawUser, userSet, rawPass, _ := u.userinfo() + if (u.scheme != "http" && u.scheme != "https") || !userSet { + return proxy, "", "" + } + host := u.hostname() + if p, ok, err := u.port(); err == nil && ok && p != 0 { + host = host + ":" + strconv.Itoa(p) + } + stripped := urlunsplit(u.scheme, host, u.path, u.query, u.fragment) + return stripped, rawUser, rawPass +} + +// ResolveWebRTCArgs replaces --fingerprint-webrtc-ip=auto with the resolved +// proxy exit IP. With no proxy or an unresolvable exit IP, the flag is dropped. +// resolveExitIP is injected so callers can stub network access in tests. +func ResolveWebRTCArgs(args []string, proxy string, resolveExitIP func(proxyURL string) string) []string { + if len(args) == 0 { + return args + } + idx := slices.Index(args, "--fingerprint-webrtc-ip=auto") + if idx == -1 { + return args + } + out := slices.Clone(args) + proxyURL := extractProxyURLString(proxy) + if proxyURL == "" { + return slices.Delete(out, idx, idx+1) + } + if exitIP := resolveExitIP(proxyURL); exitIP != "" { + out[idx] = "--fingerprint-webrtc-ip=" + exitIP + return out + } + return slices.Delete(out, idx, idx+1) +} + +// assembleProxyURL builds a proxy URL from percent-encoded credentials and host +// parts. passPresent distinguishes user@host (false) from user:@host (true, +// empty password preserves the colon). +func assembleProxyURL(scheme, host string, hasPort bool, port int, encUser, encPass string, passPresent bool, path, params, query, fragment string) string { + if strings.Contains(host, ":") { + host = "[" + host + "]" + } + userinfo := "" + switch { + case passPresent: + userinfo = encUser + ":" + encPass + "@" + case encUser != "": + userinfo = encUser + "@" + } + netloc := userinfo + host + if hasPort { + netloc += ":" + strconv.Itoa(port) + } + return urlunparse(scheme, netloc, path, params, query, fragment) +} diff --git a/internal/fingerprint/pyurl.go b/internal/fingerprint/pyurl.go new file mode 100644 index 0000000..d64c776 --- /dev/null +++ b/internal/fingerprint/pyurl.go @@ -0,0 +1,243 @@ +package fingerprint + +import ( + "errors" + "strconv" + "strings" +) + +// This file reimplements the exact subset of CPython's urllib.parse behaviour +// (urlsplit/urlparse component extraction, urlunparse reconstruction, and +// quote/unquote with safe="") that the vendored proxy helpers depend on. Byte +// parity rides on these matching CPython's urllib.parse precisely, so +// the semantics are copied verbatim rather than delegated to net/url (which +// differs in lowercasing, percent-encoding, and userinfo splitting). + +const ( + pyAlwaysSafe = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-~" + hexUpper = "0123456789ABCDEF" + schemeChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-." +) + +var errBadPort = errors.New("port out of range or non-numeric") + +// pyURL holds the six urlparse components. +type pyURL struct { + scheme string + netloc string + path string + params string + query string + fragment string +} + +// urlparse mirrors CPython urllib.parse.urlparse: urlsplit plus the trailing +// params split from the last path segment. +func urlparse(rawurl string) pyURL { + u := urlsplit(rawurl) + if strings.Contains(u.path, ";") { + u.path, u.params = splitParams(u.path) + } + return u +} + +func urlsplit(rawurl string) pyURL { + var u pyURL + url := rawurl + if i := strings.IndexByte(url, ':'); i > 0 && isASCIILetter(url[0]) && allSchemeChars(url[1:i]) { + u.scheme = strings.ToLower(url[:i]) + url = url[i+1:] + } + if strings.HasPrefix(url, "//") { + delim := len(url) + for _, c := range []byte{'/', '?', '#'} { + if w := strings.IndexByte(url[2:], c); w >= 0 && w+2 < delim { + delim = w + 2 + } + } + u.netloc = url[2:delim] + url = url[delim:] + } + if i := strings.IndexByte(url, '#'); i >= 0 { + u.fragment = url[i+1:] + url = url[:i] + } + if i := strings.IndexByte(url, '?'); i >= 0 { + u.query = url[i+1:] + url = url[:i] + } + u.path = url + return u +} + +func splitParams(path string) (string, string) { + var i int + if slash := strings.LastIndexByte(path, '/'); slash >= 0 { + i = strings.IndexByte(path[slash:], ';') + if i < 0 { + return path, "" + } + i += slash + } else { + i = strings.IndexByte(path, ';') + } + return path[:i], path[i+1:] +} + +// userinfo returns (username, usernameSet, password, passwordSet), matching +// CPython's rpartition('@') + partition(':') semantics. +func (u pyURL) userinfo() (string, bool, string, bool) { + at := strings.LastIndexByte(u.netloc, '@') + if at < 0 { + return "", false, "", false + } + ui := u.netloc[:at] + if user, pass, found := strings.Cut(ui, ":"); found { + return user, true, pass, true + } + return ui, true, "", false +} + +// hostinfo returns the raw host and port strings from netloc. +func (u pyURL) hostinfo() (string, string) { + hi := u.netloc + if at := strings.LastIndexByte(hi, '@'); at >= 0 { + hi = hi[at+1:] + } + if _, bracketed, found := strings.Cut(hi, "["); found { + host, rest, _ := strings.Cut(bracketed, "]") + _, port, _ := strings.Cut(rest, ":") + return host, port + } + host, port, _ := strings.Cut(hi, ":") + return host, port +} + +// hostname returns the lowercased host, or "" when absent (CPython returns None). +func (u pyURL) hostname() string { + host, _ := u.hostinfo() + if host == "" { + return "" + } + return strings.ToLower(host) +} + +// port returns (value, present, error); error mirrors CPython raising ValueError +// on a non-numeric or out-of-range port. +func (u pyURL) port() (int, bool, error) { + _, ps := u.hostinfo() + if ps == "" { + return 0, false, nil + } + p, err := strconv.Atoi(ps) + if err != nil { + return 0, false, errBadPort + } + if p < 0 || p > 65535 { + return 0, false, errBadPort + } + return p, true, nil +} + +func urlunparse(scheme, netloc, path, params, query, fragment string) string { + url := path + if params != "" { + url = url + ";" + params + } + return urlunsplit(scheme, netloc, url, query, fragment) +} + +func urlunsplit(scheme, netloc, url, query, fragment string) string { + if netloc != "" || (scheme != "" && schemeUsesNetloc(scheme) && !strings.HasPrefix(url, "//")) { + if url != "" && !strings.HasPrefix(url, "/") { + url = "/" + url + } + url = "//" + netloc + url + } + if scheme != "" { + url = scheme + ":" + url + } + if query != "" { + url = url + "?" + query + } + if fragment != "" { + url = url + "#" + fragment + } + return url +} + +var usesNetloc = map[string]struct{}{ + "": {}, "ftp": {}, "http": {}, "gopher": {}, "nntp": {}, "telnet": {}, + "imap": {}, "wais": {}, "file": {}, "mms": {}, "https": {}, "shttp": {}, + "snews": {}, "prospero": {}, "rtsp": {}, "rtspu": {}, "rsync": {}, "svn": {}, + "svn+ssh": {}, "sftp": {}, "nfs": {}, "git": {}, "git+ssh": {}, "ws": {}, "wss": {}, +} + +func schemeUsesNetloc(scheme string) bool { + _, ok := usesNetloc[scheme] + return ok +} + +// pyQuote percent-encodes every byte outside CPython's always-safe set, with +// uppercase hex, matching urllib.parse.quote(s, safe=""). +func pyQuote(s string) string { + var b strings.Builder + for i := range len(s) { + c := s[i] + if strings.IndexByte(pyAlwaysSafe, c) >= 0 { + b.WriteByte(c) + continue + } + b.WriteByte('%') + b.WriteByte(hexUpper[c>>4]) + b.WriteByte(hexUpper[c&0xf]) + } + return b.String() +} + +// pyUnquote decodes %XX escapes, leaving malformed escapes literal, matching +// urllib.parse.unquote for ASCII input. +func pyUnquote(s string) string { + if !strings.Contains(s, "%") { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+2 <= len(s)-1 { + hi, ok1 := unhex(s[i+1]) + lo, ok2 := unhex(s[i+2]) + if ok1 && ok2 { + b.WriteByte(hi<<4 | lo) + i += 2 + continue + } + } + b.WriteByte(s[i]) + } + return b.String() +} + +func unhex(c byte) (byte, bool) { + switch { + case c >= '0' && c <= '9': + return c - '0', true + case c >= 'a' && c <= 'f': + return c - 'a' + 10, true + case c >= 'A' && c <= 'F': + return c - 'A' + 10, true + } + return 0, false +} + +func isASCIILetter(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +func allSchemeChars(s string) bool { + for i := range len(s) { + if strings.IndexByte(schemeChars, s[i]) < 0 { + return false + } + } + return true +} diff --git a/internal/fingerprint/testdata/golden.json b/internal/fingerprint/testdata/golden.json new file mode 100644 index 0000000..0bc209a --- /dev/null +++ b/internal/fingerprint/testdata/golden.json @@ -0,0 +1,1253 @@ +{ + "exit_ip_stub": "203.0.113.7", + "country_locale_map": { + "US": "en-US", + "GB": "en-GB", + "AU": "en-AU", + "CA": "en-CA", + "NZ": "en-NZ", + "IE": "en-IE", + "ZA": "en-ZA", + "SG": "en-SG", + "DE": "de-DE", + "AT": "de-AT", + "CH": "de-CH", + "FR": "fr-FR", + "BE": "fr-BE", + "ES": "es-ES", + "MX": "es-MX", + "AR": "es-AR", + "CO": "es-CO", + "CL": "es-CL", + "BR": "pt-BR", + "PT": "pt-PT", + "IT": "it-IT", + "NL": "nl-NL", + "JP": "ja-JP", + "KR": "ko-KR", + "CN": "zh-CN", + "TW": "zh-TW", + "HK": "zh-HK", + "RU": "ru-RU", + "UA": "uk-UA", + "PL": "pl-PL", + "CZ": "cs-CZ", + "RO": "ro-RO", + "IL": "he-IL", + "TR": "tr-TR", + "SA": "ar-SA", + "AE": "ar-AE", + "EG": "ar-EG", + "IN": "hi-IN", + "ID": "id-ID", + "PH": "en-PH", + "TH": "th-TH", + "VN": "vi-VN", + "MY": "ms-MY", + "SE": "sv-SE", + "NO": "nb-NO", + "DK": "da-DK", + "FI": "fi-FI", + "GR": "el-GR", + "HU": "hu-HU", + "BG": "bg-BG", + "SI": "sl-SI", + "SK": "sk-SK", + "HR": "hr-HR", + "RS": "sr-RS", + "LT": "lt-LT", + "LV": "lv-LV", + "EE": "et-EE", + "IS": "is-IS", + "LU": "fr-LU", + "MT": "en-MT", + "CY": "el-CY", + "MD": "ro-MD", + "BY": "ru-BY", + "GE": "ka-GE", + "AL": "sq-AL", + "MK": "mk-MK", + "BA": "bs-BA", + "PE": "es-PE", + "VE": "es-VE", + "EC": "es-EC", + "UY": "es-UY", + "CR": "es-CR", + "DO": "es-DO", + "GT": "es-GT", + "BO": "es-BO", + "PY": "es-PY", + "PK": "en-PK", + "BD": "bn-BD", + "LK": "si-LK", + "KZ": "ru-KZ", + "IR": "fa-IR", + "IQ": "ar-IQ", + "JO": "ar-JO", + "LB": "ar-LB", + "KW": "ar-KW", + "QA": "ar-QA", + "OM": "ar-OM", + "BH": "ar-BH", + "NG": "en-NG", + "KE": "en-KE", + "MA": "fr-MA", + "DZ": "ar-DZ", + "TN": "ar-TN", + "GH": "en-GH", + "AM": "hy-AM", + "AZ": "az-AZ", + "UZ": "uz-UZ", + "KG": "ky-KG", + "TJ": "tg-TJ", + "TM": "tk-TM", + "ME": "sr-ME", + "XK": "sq-XK", + "LI": "de-LI", + "MC": "fr-MC", + "AD": "ca-AD", + "MM": "my-MM", + "KH": "km-KH", + "LA": "lo-LA", + "MN": "mn-MN", + "BN": "ms-BN", + "MO": "zh-MO", + "YE": "ar-YE", + "SY": "ar-SY", + "PS": "ar-PS", + "LY": "ar-LY", + "ET": "am-ET", + "TZ": "sw-TZ", + "UG": "en-UG", + "SN": "fr-SN", + "CI": "fr-CI", + "CM": "fr-CM", + "AO": "pt-AO", + "MZ": "pt-MZ", + "ZM": "en-ZM", + "ZW": "en-ZW", + "HN": "es-HN", + "NI": "es-NI", + "SV": "es-SV", + "PA": "es-PA", + "JM": "en-JM", + "TT": "en-TT", + "PR": "es-PR" + }, + "default_stealth_args": [ + { + "system": "Linux", + "seed": 55555, + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows" + ] + }, + { + "system": "Darwin", + "seed": 55555, + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=macos" + ] + }, + { + "system": "Windows", + "seed": 55555, + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows" + ] + } + ], + "ensure_proxy_scheme": [ + { + "input": "proxy.example:8080", + "output": "http://proxy.example:8080" + }, + { + "input": "http://proxy.example:8080", + "output": "http://proxy.example:8080" + }, + { + "input": "socks5://proxy.example:1080", + "output": "socks5://proxy.example:1080" + } + ], + "normalize_socks": [ + { + "input": "http://proxy.example:8080", + "output": "http://proxy.example:8080" + }, + { + "input": "socks5://proxy.example:1080", + "output": "socks5://proxy.example:1080" + }, + { + "input": "http://user:p%40ss@proxy.example:8080", + "output": "http://user:p%40ss@proxy.example:8080" + }, + { + "input": "socks5://user:p%40ss@proxy.example:1080", + "output": "socks5://user:p%40ss@proxy.example:1080" + }, + { + "input": "socks5://user:p@ss=word@proxy.example:1080", + "output": "socks5://user:p%40ss%3Dword@proxy.example:1080" + }, + { + "input": "socks5://USER:P@SS@HOST.example:1080", + "output": "socks5://USER:P%40SS@host.example:1080" + }, + { + "input": "socks5://user:@proxy.example:1080", + "output": "socks5://user:@proxy.example:1080" + }, + { + "input": "socks5://user@proxy.example:1080", + "output": "socks5://user@proxy.example:1080" + }, + { + "input": "socks5://user:pass@[2001:db8::1]:1080", + "output": "socks5://user:pass@[2001:db8::1]:1080" + }, + { + "input": "socks5://proxy.example:not-a-port", + "output": "socks5://proxy.example:not-a-port" + } + ], + "split_proxy_auth": [ + { + "input": "http://bob:secret@proxy.example:8080", + "server": "http://proxy.example:8080", + "username": "bob", + "password": "secret" + }, + { + "input": "http://bob:secret@Proxy.EXAMPLE:8080", + "server": "http://proxy.example:8080", + "username": "bob", + "password": "secret" + }, + { + "input": "http://user%40host:p%40ss@host.example:8080", + "server": "http://host.example:8080", + "username": "user%40host", + "password": "p%40ss" + }, + { + "input": "https://u:p@[2001:db8::1]:8443", + "server": "https://2001:db8::1:8443", + "username": "u", + "password": "p" + }, + { + "input": "http://bob@proxy.example:8080", + "server": "http://proxy.example:8080", + "username": "bob", + "password": "" + }, + { + "input": "http://user:@proxy.example:8080", + "server": "http://proxy.example:8080", + "username": "user", + "password": "" + }, + { + "input": "http://proxy.example:8080", + "server": "http://proxy.example:8080", + "username": "", + "password": "" + }, + { + "input": "socks5://user:pass@proxy.example:1080", + "server": "socks5://user:pass@proxy.example:1080", + "username": "", + "password": "" + } + ], + "fork_parity_args": [ + { + "locale": "", + "proxy": null, + "output": [ + "--fingerprint-platform=windows", + "--fingerprint-platform-version=19.0.0", + "--fingerprint-brand=Chrome", + "--fingerprint-brand-version=148.0.0.0", + "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", + "--fingerprint-fonts-dir=/opt/winfonts", + "--fingerprinting-client-rects-noise", + "--fingerprinting-canvas-measuretext-noise", + "--fingerprinting-canvas-image-data-noise", + "--accept-lang=en-US,en" + ] + }, + { + "locale": "en-US", + "proxy": null, + "output": [ + "--fingerprint-platform=windows", + "--fingerprint-platform-version=19.0.0", + "--fingerprint-brand=Chrome", + "--fingerprint-brand-version=148.0.0.0", + "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", + "--fingerprint-fonts-dir=/opt/winfonts", + "--fingerprinting-client-rects-noise", + "--fingerprinting-canvas-measuretext-noise", + "--fingerprinting-canvas-image-data-noise", + "--accept-lang=en-US,en" + ] + }, + { + "locale": "de-DE", + "proxy": "http://p:1", + "output": [ + "--fingerprint-platform=windows", + "--fingerprint-platform-version=19.0.0", + "--fingerprint-brand=Chrome", + "--fingerprint-brand-version=148.0.0.0", + "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", + "--fingerprint-fonts-dir=/opt/winfonts", + "--fingerprinting-client-rects-noise", + "--fingerprinting-canvas-measuretext-noise", + "--fingerprinting-canvas-image-data-noise", + "--accept-lang=de-DE,de", + "--fingerprint-network-profile=residential" + ] + }, + { + "locale": "fr", + "proxy": "socks5://p:1", + "output": [ + "--fingerprint-platform=windows", + "--fingerprint-platform-version=19.0.0", + "--fingerprint-brand=Chrome", + "--fingerprint-brand-version=148.0.0.0", + "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", + "--fingerprint-fonts-dir=/opt/winfonts", + "--fingerprinting-client-rects-noise", + "--fingerprinting-canvas-measuretext-noise", + "--fingerprinting-canvas-image-data-noise", + "--accept-lang=fr", + "--fingerprint-network-profile=residential" + ] + }, + { + "locale": "", + "proxy": "http://p:1", + "output": [ + "--fingerprint-platform=windows", + "--fingerprint-platform-version=19.0.0", + "--fingerprint-brand=Chrome", + "--fingerprint-brand-version=148.0.0.0", + "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", + "--fingerprint-fonts-dir=/opt/winfonts", + "--fingerprinting-client-rects-noise", + "--fingerprinting-canvas-measuretext-noise", + "--fingerprinting-canvas-image-data-noise", + "--accept-lang=en-US,en", + "--fingerprint-network-profile=residential" + ] + } + ], + "resolve_webrtc": [ + { + "input_args": [ + "--fingerprint=1", + "--no-sandbox" + ], + "proxy": "http://proxy.example:8080", + "exit_ip": "203.0.113.7", + "output": [ + "--fingerprint=1", + "--no-sandbox" + ] + }, + { + "input_args": [ + "--fingerprint-webrtc-ip=auto" + ], + "proxy": "http://proxy.example:8080", + "exit_ip": "203.0.113.7", + "output": [ + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "input_args": [ + "--fingerprint-webrtc-ip=auto" + ], + "proxy": "socks5://proxy.example:1080", + "exit_ip": "203.0.113.7", + "output": [ + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "input_args": [ + "--fingerprint-webrtc-ip=auto" + ], + "proxy": null, + "exit_ip": "203.0.113.7", + "output": [] + }, + { + "input_args": [ + "--x", + "--fingerprint-webrtc-ip=auto" + ], + "proxy": "http://proxy.example:8080", + "exit_ip": null, + "output": [ + "--x" + ] + } + ], + "build_args": [ + { + "name": "headed-adds-gpu-flag", + "input": { + "stealth_args": true, + "extra_args": [ + "--fingerprint=1" + ], + "headless": false + }, + "output": [ + "--no-sandbox", + "--fingerprint=1", + "--fingerprint-platform=windows", + "--ignore-gpu-blocklist" + ] + }, + { + "name": "timezone-only", + "input": { + "stealth_args": true, + "extra_args": null, + "timezone": "Europe/Berlin" + }, + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--fingerprint-timezone=Europe/Berlin" + ] + }, + { + "name": "locale-only", + "input": { + "stealth_args": true, + "extra_args": null, + "locale": "de-DE" + }, + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--lang=de-DE", + "--fingerprint-locale=de-DE" + ] + }, + { + "name": "no-stealth", + "input": { + "stealth_args": false, + "extra_args": [ + "--foo=bar" + ], + "timezone": "UTC", + "locale": "en-US" + }, + "output": [ + "--foo=bar", + "--fingerprint-timezone=UTC", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "name": "override-fingerprint", + "input": { + "stealth_args": true, + "extra_args": [ + "--fingerprint=99999", + "--proxy-server=http://h:1" + ] + }, + "output": [ + "--no-sandbox", + "--fingerprint=99999", + "--fingerprint-platform=windows", + "--proxy-server=http://h:1" + ] + }, + { + "name": "extensions", + "input": { + "stealth_args": true, + "extra_args": [ + "--fingerprint=1" + ], + "extension_paths": [ + "/opt/ext/a", + "/opt/ext/b" + ] + }, + "output": [ + "--no-sandbox", + "--fingerprint=1", + "--fingerprint-platform=windows", + "--load-extension=/opt/ext/a,/opt/ext/b", + "--disable-extensions-except=/opt/ext/a,/opt/ext/b" + ] + }, + { + "name": "start-maximized", + "input": { + "stealth_args": true, + "extra_args": [ + "--fingerprint=1" + ], + "start_maximized": true + }, + "output": [ + "--no-sandbox", + "--fingerprint=1", + "--fingerprint-platform=windows", + "--start-maximized" + ] + }, + { + "name": "start-maximized-suppressed", + "input": { + "stealth_args": true, + "extra_args": [ + "--fingerprint=1", + "--window-size=800,600" + ], + "start_maximized": true + }, + "output": [ + "--no-sandbox", + "--fingerprint=1", + "--fingerprint-platform=windows", + "--window-size=800,600" + ] + } + ], + "compose_argv": [ + { + "seed": null, + "proxy": null, + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows" + ] + }, + { + "seed": null, + "proxy": null, + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows" + ] + }, + { + "seed": null, + "proxy": null, + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": null, + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "http://proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080" + ] + }, + { + "seed": null, + "proxy": "http://proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": null, + "proxy": "http://proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "http://proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "socks5://proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080" + ] + }, + { + "seed": null, + "proxy": "socks5://proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": null, + "proxy": "socks5://proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "socks5://proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080" + ] + }, + { + "seed": null, + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": null, + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": null, + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=55555", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": null, + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows" + ] + }, + { + "seed": "12345", + "proxy": null, + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows" + ] + }, + { + "seed": "12345", + "proxy": null, + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": null, + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "http://proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080" + ] + }, + { + "seed": "12345", + "proxy": "http://proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": "12345", + "proxy": "http://proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "http://proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "socks5://proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080" + ] + }, + { + "seed": "12345", + "proxy": "socks5://proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": "12345", + "proxy": "socks5://proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "socks5://proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080" + ] + }, + { + "seed": "12345", + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": "12345", + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "http://user:p%40ss@proxy.example:8080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=http://proxy.example:8080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p%40ss@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": null, + "locale": null, + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "none", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + }, + { + "seed": "12345", + "proxy": "socks5://user:p@ss=word@proxy.example:1080", + "timezone": "America/New_York", + "locale": "en-US", + "webrtc": "auto", + "output": [ + "--no-sandbox", + "--fingerprint=12345", + "--fingerprint-platform=windows", + "--proxy-server=socks5://user:p%40ss%3Dword@proxy.example:1080", + "--fingerprint-webrtc-ip=203.0.113.7", + "--fingerprint-timezone=America/New_York", + "--lang=en-US", + "--fingerprint-locale=en-US" + ] + } + ] +} diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go new file mode 100644 index 0000000..112d2e5 --- /dev/null +++ b/internal/mcp/mcp.go @@ -0,0 +1,124 @@ +// Package mcp installs a CDP browser driver and generates the MCP client config +// that points it at cuttle's browser for a given context and profile seed. +package mcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/glim-sh/cuttle/internal/xdg" +) + +// Driver describes a CDP driver that exposes an MCP server. Names and install +// commands mirror the CLI's driver briefing table; MCPCommand/Args/EnvVar carry +// the extra bit the briefing does not: how to run the driver as an MCP server +// and which env var points it at a CDP endpoint. +type Driver struct { + Name string + Install string + MCPCommand string + MCPArgs []string + EnvVar string +} + +// DefaultDriver is used when `cuttle mcp` is invoked without a driver argument. +const DefaultDriver = "browser-use" + +var drivers = map[string]Driver{ + DefaultDriver: { + Name: DefaultDriver, + Install: "uv tool install browser-use", + MCPCommand: "browser-use", + MCPArgs: []string{"--mcp"}, + EnvVar: "BU_CDP_URL", + }, +} + +var ( + errUnknownDriver = errors.New("unknown MCP driver") + errInstallFailed = errors.New("driver install failed") +) + +// Lookup returns the driver metadata for name. +func Lookup(name string) (Driver, error) { + d, ok := drivers[name] + if !ok { + return Driver{}, fmt.Errorf("%w %q (known: browser-use)", errUnknownDriver, name) + } + return d, nil +} + +type serverConfig struct { + Command string `json:"command"` + Args []string `json:"args,omitempty"` + Env map[string]string `json:"env,omitempty"` +} + +// Config is the MCP client config: a map of server name to its launch spec. +type Config struct { + MCPServers map[string]serverConfig `json:"mcpServers"` +} + +// BuildConfig produces the MCP config for a driver pointed at cdpURL (which +// already carries any ?fingerprint=). +func BuildConfig(d Driver, cdpURL string) Config { + return Config{ + MCPServers: map[string]serverConfig{ + d.Name: { + Command: d.MCPCommand, + Args: d.MCPArgs, + Env: map[string]string{d.EnvVar: cdpURL}, + }, + }, + } +} + +// Marshal renders the config as indented JSON. +func Marshal(c Config) ([]byte, error) { + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + return nil, fmt.Errorf("encoding MCP config: %w", err) + } + return b, nil +} + +// EnsureInstalled installs the driver via its install command when it is not +// already on PATH. It is a no-op when the driver is present. +func EnsureInstalled(ctx context.Context, d Driver) error { + if _, err := exec.LookPath(d.MCPCommand); err == nil { + return nil + } + fields := strings.Fields(d.Install) + if len(fields) == 0 { + return fmt.Errorf("%w: no install command for %s", errInstallFailed, d.Name) + } + cmd := exec.CommandContext(ctx, fields[0], fields[1:]...) //nolint:gosec // install command is a fixed table entry + cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("%w (%s): %w", errInstallFailed, d.Install, err) + } + return nil +} + +// DefaultConfigPath is where a driver's MCP config is written when no explicit +// path is given: $XDG_CONFIG_HOME/cuttle/mcp/.json. +func DefaultConfigPath(driver string) string { + return filepath.Join(xdg.ConfigDir(), "cuttle", "mcp", driver+".json") +} + +// WriteConfig writes the rendered config to path, creating parent dirs. +func WriteConfig(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("creating MCP config dir: %w", err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("writing MCP config: %w", err) + } + return nil +} diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go new file mode 100644 index 0000000..571a477 --- /dev/null +++ b/internal/mcp/mcp_test.go @@ -0,0 +1,48 @@ +package mcp + +import ( + "errors" + "testing" +) + +func TestBuildConfigGolden(t *testing.T) { + t.Parallel() + d, err := Lookup(DefaultDriver) + if err != nil { + t.Fatalf("Lookup: %v", err) + } + got, err := Marshal(BuildConfig(d, "http://127.0.0.1:9222?fingerprint=linkedin")) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + want := `{ + "mcpServers": { + "browser-use": { + "command": "browser-use", + "args": [ + "--mcp" + ], + "env": { + "BU_CDP_URL": "http://127.0.0.1:9222?fingerprint=linkedin" + } + } + } +}` + if string(got) != want { + t.Fatalf("config JSON mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestLookupUnknown(t *testing.T) { + t.Parallel() + if _, err := Lookup("nope"); !errors.Is(err, errUnknownDriver) { + t.Fatalf("want errUnknownDriver, got %v", err) + } +} + +func TestDefaultConfigPath(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", "/cfg") + if got := DefaultConfigPath("browser-use"); got != "/cfg/cuttle/mcp/browser-use.json" { + t.Fatalf("path=%q", got) + } +} diff --git a/internal/profile/lock.go b/internal/profile/lock.go new file mode 100644 index 0000000..1aef5df --- /dev/null +++ b/internal/profile/lock.go @@ -0,0 +1,112 @@ +package profile + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +// errCheckedOut is returned when a profile is already held by a live session +// elsewhere. A profile maps to a single browser seed, and one seed cannot be +// driven by two sessions at once (Chrome's own single-writer rule). +var errCheckedOut = errors.New("profile is checked out by another live session") + +const lockName = "session.lock" + +// lockInfo is what a held lock records, so a stale lock (from a crashed session) +// can be detected and stolen. +type lockInfo struct { + PID int `json:"pid"` + Host string `json:"host"` + Acquired time.Time `json:"acquired"` +} + +// lock is a held single-writer lock over a profile directory. +type lock struct { + path string +} + +// acquireLock takes the profile's single-writer lock. If the lock file exists +// and its owning process is still alive it fails with errCheckedOut; a stale +// lock (owner gone) is removed and retaken. +func acquireLock(dir string) (*lock, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("creating profile dir: %w", err) + } + path := filepath.Join(dir, lockName) + for range 2 { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err == nil { + host, _ := os.Hostname() + info := lockInfo{PID: os.Getpid(), Host: host, Acquired: time.Now().UTC()} + data, _ := json.Marshal(info) + if _, werr := f.Write(data); werr != nil { + _ = f.Close() + _ = os.Remove(path) + return nil, fmt.Errorf("writing lock: %w", werr) + } + if cerr := f.Close(); cerr != nil { + return nil, fmt.Errorf("closing lock: %w", cerr) + } + return &lock{path: path}, nil + } + if !errors.Is(err, os.ErrExist) { + return nil, fmt.Errorf("opening lock: %w", err) + } + if held := lockHeld(path); held { + return nil, errCheckedOut + } + // Stale lock from a dead session: remove and retry once. + if rmErr := os.Remove(path); rmErr != nil && !errors.Is(rmErr, os.ErrNotExist) { + return nil, fmt.Errorf("clearing stale lock: %w", rmErr) + } + } + return nil, errCheckedOut +} + +// release removes the lock file. It is safe to call more than once. +func (l *lock) release() error { + if l == nil { + return nil + } + if err := os.Remove(l.path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("releasing lock: %w", err) + } + return nil +} + +// lockHeld reports whether the lock file names a live process on this host. A +// malformed lock, a cross-host lock, or a dead PID is treated as not held (stale) +// so a crash never wedges a profile permanently. +func lockHeld(path string) bool { + data, err := os.ReadFile(path) + if err != nil { + return false + } + var info lockInfo + if err := json.Unmarshal(data, &info); err != nil { + return false + } + host, _ := os.Hostname() + if info.Host != "" && host != "" && info.Host != host { + // Owned on a different machine; we cannot check liveness, so do not steal. + return true + } + return pidAlive(info.PID) +} + +func pidAlive(pid int) bool { + if pid <= 0 { + return false + } + p, err := os.FindProcess(pid) + if err != nil { + return false + } + err = p.Signal(syscall.Signal(0)) + return err == nil || errors.Is(err, syscall.EPERM) +} diff --git a/internal/profile/lock_test.go b/internal/profile/lock_test.go new file mode 100644 index 0000000..b34c337 --- /dev/null +++ b/internal/profile/lock_test.go @@ -0,0 +1,92 @@ +package profile + +import ( + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +func TestLockSingleWriter(t *testing.T) { + t.Parallel() + dir := t.TempDir() + lk, err := acquireLock(dir) + if err != nil { + t.Fatalf("first acquire: %v", err) + } + if _, serr := acquireLock(dir); !errors.Is(serr, errCheckedOut) { + t.Fatalf("second acquire: want errCheckedOut, got %v", serr) + } + if rerr := lk.release(); rerr != nil { + t.Fatalf("release: %v", rerr) + } + lk2, err := acquireLock(dir) + if err != nil { + t.Fatalf("re-acquire after release: %v", err) + } + _ = lk2.release() +} + +func TestLockStealsStaleLock(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, lockName) + host, _ := os.Hostname() + info := lockInfo{PID: deadPID(t), Host: host, Acquired: time.Now().UTC()} + data, _ := json.Marshal(info) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("seed stale lock: %v", err) + } + if lockHeld(path) { + t.Fatal("dead-PID lock should read as not held") + } + lk, err := acquireLock(dir) + if err != nil { + t.Fatalf("acquire over stale lock: %v", err) + } + _ = lk.release() +} + +func TestLockHeldForLiveProcess(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, lockName) + host, _ := os.Hostname() + data, _ := json.Marshal(lockInfo{PID: os.Getpid(), Host: host, Acquired: time.Now().UTC()}) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("seed lock: %v", err) + } + if !lockHeld(path) { + t.Fatal("live-PID lock should read as held") + } +} + +func TestLockHeldForeignHost(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, lockName) + data, _ := json.Marshal(lockInfo{PID: deadPID(t), Host: "some-other-host", Acquired: time.Now().UTC()}) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("seed lock: %v", err) + } + // A lock owned on a different machine cannot have its liveness checked, so it + // is treated as held rather than stolen. + if !lockHeld(path) { + t.Fatal("foreign-host lock should read as held") + } +} + +// deadPID returns the PID of a process that has already exited. +func deadPID(t *testing.T) int { + t.Helper() + cmd := exec.CommandContext(t.Context(), "/bin/sh", "-c", "exit 0") + if err := cmd.Start(); err != nil { + t.Fatalf("spawn: %v", err) + } + pid := cmd.Process.Pid + _ = cmd.Wait() + return pid +} diff --git a/internal/profile/profile.go b/internal/profile/profile.go new file mode 100644 index 0000000..25baf0c --- /dev/null +++ b/internal/profile/profile.go @@ -0,0 +1,158 @@ +// Package profile keeps a browser profile's auth state (cookies + per-origin +// localStorage) canonical on the local machine and checks it in and out of an +// otherwise-ephemeral remote browser seed over CDP. +// +// A named profile is a cuttle seed. Its storage_state.json lives under +// $XDG_DATA_HOME/cuttle/profiles//. On session start the state is injected +// into the fresh remote seed; during and at the end of the session it is +// extracted back and written atomically, so a crash loses at most one checkpoint +// interval of cookie deltas. "Resides locally" is true at rest; a live session +// necessarily holds the cookies on the remote to act as the user. +package profile + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/glim-sh/cuttle/internal/cdp" + "github.com/glim-sh/cuttle/internal/fingerprint" + "github.com/glim-sh/cuttle/internal/xdg" +) + +var ( + errInvalidName = errors.New("invalid profile name (allowed: letters, digits, '_' and '-', 1-128 chars)") + errReserved = errors.New("profile name is reserved") +) + +// ValidName reports whether name is a legal profile name. A profile name is a +// cuttle seed, so it shares the seed grammar (fingerprint.ValidSeed). +func ValidName(name string) bool { + return fingerprint.ValidSeed(name) +} + +func checkName(name string) error { + if name == fingerprint.ReservedSeed { + return fmt.Errorf("%w: %q", errReserved, name) + } + if !fingerprint.ValidSeed(name) { + return fmt.Errorf("%w: %q", errInvalidName, name) + } + return nil +} + +// DataDir is $XDG_DATA_HOME/cuttle/profiles/, falling back to +// ~/.local/share. +func DataDir(name string) string { + return filepath.Join(xdg.DataDir(), "cuttle", "profiles", name) +} + +func statePath(dir string) string { return filepath.Join(dir, "storage_state.json") } + +// loadState reads a profile's storage_state.json. A missing file yields an empty +// state (a brand-new profile), not an error. +func loadState(dir string) (*cdp.StorageState, error) { + data, err := os.ReadFile(statePath(dir)) + if errors.Is(err, os.ErrNotExist) { + return &cdp.StorageState{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading profile state: %w", err) + } + st := &cdp.StorageState{} + if err := json.Unmarshal(data, st); err != nil { + return nil, fmt.Errorf("parsing profile state: %w", err) + } + return st, nil +} + +// saveState writes storage_state.json atomically (temp file in the same dir then +// rename) so a crash mid-write never leaves a truncated profile. +func saveState(dir string, st *cdp.StorageState) error { + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("creating profile dir: %w", err) + } + data, err := json.MarshalIndent(st, "", " ") + if err != nil { + return fmt.Errorf("encoding profile state: %w", err) + } + tmp, err := os.CreateTemp(dir, ".storage_state.*.tmp") + if err != nil { + return fmt.Errorf("creating temp state: %w", err) + } + tmpName := tmp.Name() + defer func() { _ = os.Remove(tmpName) }() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return fmt.Errorf("writing temp state: %w", err) + } + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return fmt.Errorf("chmod temp state: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("closing temp state: %w", err) + } + if err := os.Rename(tmpName, statePath(dir)); err != nil { + return fmt.Errorf("committing profile state: %w", err) + } + return nil +} + +// carryForwardLocalStorage preserves the last-known localStorage for origins +// that failed to LOAD during an extract, so a transient per-origin blip does not +// drop that origin's persisted localStorage when saveState overwrites the +// canonical file. An origin that loaded but was genuinely empty (e.g. a real +// logout) is not in failed, so its state is still correctly cleared. A missing +// or unreadable prior file is treated as "nothing to carry forward". +func carryForwardLocalStorage(dir string, st *cdp.StorageState, failed []string) *cdp.StorageState { + prior, err := loadState(dir) + if err != nil { + return st + } + priorByOrigin := make(map[string]cdp.Origin, len(prior.Origins)) + for _, o := range prior.Origins { + priorByOrigin[o.Origin] = o + } + for _, origin := range failed { + if o, ok := priorByOrigin[origin]; ok { + st.Origins = append(st.Origins, o) + } + } + return st +} + +// candidateOrigins is the set of origins a checkin re-reads localStorage from: +// origins already recorded in the state, plus https origins derived from cookie +// domains, so a fresh login's localStorage is captured even before its origin is +// first recorded. localStorage is origin-scoped, so unknown origins cannot be +// discovered without visiting them. +func candidateOrigins(st *cdp.StorageState) []string { + seen := map[string]struct{}{} + var out []string + add := func(o string) { + if o == "" { + return + } + if _, ok := seen[o]; ok { + return + } + seen[o] = struct{}{} + out = append(out, o) + } + for _, o := range st.OriginURLs() { + add(o) + } + for _, c := range st.Cookies { + host := strings.TrimPrefix(c.Domain, ".") + if host == "" { + continue + } + add((&url.URL{Scheme: "https", Host: host}).String()) + } + return out +} diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go new file mode 100644 index 0000000..7a9f477 --- /dev/null +++ b/internal/profile/profile_test.go @@ -0,0 +1,85 @@ +package profile + +import ( + "slices" + "testing" + + "github.com/glim-sh/cuttle/internal/cdp" +) + +const ( + exampleDomain = "example.com" + exampleOrigin = "https://example.com" +) + +func TestValidName(t *testing.T) { + t.Parallel() + tests := []struct { + name string + want bool + }{ + {"default", true}, + {"linkedin", true}, + {"a_b-C9", true}, + {"", false}, + {"__default__", false}, + {"has space", false}, + {"dots.bad", false}, + {"slash/bad", false}, + } + for _, tt := range tests { + if got := ValidName(tt.name); got != tt.want { + t.Errorf("ValidName(%q)=%v want %v", tt.name, got, tt.want) + } + } +} + +func TestStateRoundTrip(t *testing.T) { + t.Parallel() + dir := t.TempDir() + want := &cdp.StorageState{ + Cookies: []cdp.Cookie{{Name: "sid", Value: "v", Domain: exampleDomain, Path: "/", Expires: -1}}, + Origins: []cdp.Origin{{Origin: exampleOrigin, LocalStorage: []cdp.LocalStorageItem{{Name: "k", Value: "1"}}}}, + } + if err := saveState(dir, want); err != nil { + t.Fatalf("saveState: %v", err) + } + got, err := loadState(dir) + if err != nil { + t.Fatalf("loadState: %v", err) + } + if len(got.Cookies) != 1 || got.Cookies[0].Name != "sid" { + t.Fatalf("cookies: %+v", got.Cookies) + } + if len(got.Origins) != 1 || got.Origins[0].Origin != exampleOrigin { + t.Fatalf("origins: %+v", got.Origins) + } +} + +func TestLoadStateMissing(t *testing.T) { + t.Parallel() + st, err := loadState(t.TempDir()) + if err != nil { + t.Fatalf("missing state should not error: %v", err) + } + if len(st.Cookies) != 0 || len(st.Origins) != 0 { + t.Fatalf("expected empty state, got %+v", st) + } +} + +func TestCandidateOrigins(t *testing.T) { + t.Parallel() + st := &cdp.StorageState{ + Cookies: []cdp.Cookie{ + {Name: "a", Domain: ".example.com"}, + {Name: "b", Domain: "sub.test.org"}, + {Name: "c", Domain: ".example.com"}, // duplicate origin + }, + Origins: []cdp.Origin{{Origin: exampleOrigin}}, + } + got := candidateOrigins(st) + want := []string{exampleOrigin, "https://sub.test.org"} + if !slices.Equal(got, want) { + t.Fatalf("candidateOrigins=%v want %v", got, want) + } +} diff --git a/internal/profile/session.go b/internal/profile/session.go new file mode 100644 index 0000000..e803e65 --- /dev/null +++ b/internal/profile/session.go @@ -0,0 +1,165 @@ +package profile + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/glim-sh/cuttle/internal/cdp" +) + +// defaultCheckpointInterval is how often a live local-canonical session writes +// the snowballed cookie/localStorage delta back, so a crash before checkin loses +// at most this much. +const defaultCheckpointInterval = 5 * time.Minute + +// checkpointTimeout bounds one extract+save so a wedged browser cannot stall the +// checkpoint loop forever. +const checkpointTimeout = 30 * time.Second + +// Options configures a profile session. +type Options struct { + Name string // profile name == seed + CDPBase string // e.g. http://127.0.0.1:9222 + Remote bool // storage = "remote": durable remote dir, no checkout/checkin + Interval time.Duration // checkpoint interval; 0 uses the default +} + +// injectFunc and extractFunc are the CDP operations, injectable so the session +// is testable without a real browser. +type ( + injectFunc func(ctx context.Context, cdpBase, seed string, st *cdp.StorageState) error + extractFunc func(ctx context.Context, cdpBase, seed string, origins []string) (*cdp.StorageState, []string, error) +) + +// Session is a live checkout of a local-canonical profile into a remote seed. +// For a remote-persistent profile it is an inert handle (the remote dir is +// durable, so there is nothing to check out or in). +type Session struct { + opts Options + dir string + lock *lock + origins []string + + inject injectFunc + extract extractFunc + + loopCancel context.CancelFunc + loopDone chan struct{} + + closeOnce sync.Once + closeErr error +} + +// Checkout starts a session: for a local-canonical profile it takes the +// single-writer lock, injects the local storage_state into the fresh remote +// seed, and begins periodic checkpoints. For a remote-persistent profile it +// returns an inert session. The caller MUST Close the returned session (a +// deferred Close plus a signal-aware context covers Ctrl-C). +func Checkout(ctx context.Context, opts Options) (*Session, error) { + return checkoutSession(ctx, opts, cdp.Inject, cdp.Extract) +} + +func checkoutSession(ctx context.Context, opts Options, inject injectFunc, extract extractFunc) (*Session, error) { + if err := checkName(opts.Name); err != nil { + return nil, err + } + s := &Session{opts: opts, dir: DataDir(opts.Name), inject: inject, extract: extract} + if opts.Remote { + return s, nil + } + + lk, err := acquireLock(s.dir) + if err != nil { + return nil, err + } + s.lock = lk + + st, err := loadState(s.dir) + if err != nil { + _ = lk.release() + return nil, err + } + s.origins = candidateOrigins(st) + if err := inject(ctx, opts.CDPBase, opts.Name, st); err != nil { + _ = lk.release() + return nil, err + } + + s.startCheckpoints() + return s, nil +} + +func (s *Session) interval() time.Duration { + if s.opts.Interval > 0 { + return s.opts.Interval + } + return defaultCheckpointInterval +} + +// startCheckpoints runs the checkpoint loop on a detached context so a +// request-scoped timeout on the checkout ctx cannot kill periodic saves; Close +// stops it. +func (s *Session) startCheckpoints() { + ctx, cancel := context.WithCancel(context.Background()) + s.loopCancel = cancel + s.loopDone = make(chan struct{}) + go func() { + defer close(s.loopDone) + runCheckpoints(ctx, s.interval(), func() { _ = s.checkpoint() }) + }() +} + +// checkpoint extracts the current storage_state and writes it back atomically. +// It is best-effort: an extract failure (e.g. a transiently unreachable browser) +// is returned for logging but never aborts the session. Origins that failed to +// load this pass keep their last-known localStorage (carryForwardLocalStorage), +// so a transient blip does not silently drop persisted state from the canonical +// file on the unconditional overwrite. +func (s *Session) checkpoint() error { + ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) + defer cancel() + st, failed, err := s.extract(ctx, s.opts.CDPBase, s.opts.Name, s.origins) + if err != nil { + return err + } + if len(failed) > 0 { + st = carryForwardLocalStorage(s.dir, st, failed) + } + return saveState(s.dir, st) +} + +// Close checks the session in: it stops the checkpoint loop, performs a final +// extract+save, and releases the lock. Releasing the lock always happens even if +// the final save fails, so a crashed browser never wedges the profile. Close is +// idempotent. +func (s *Session) Close() error { + s.closeOnce.Do(func() { + if s.opts.Remote { + return + } + if s.loopCancel != nil { + s.loopCancel() + <-s.loopDone + } + saveErr := s.checkpoint() + relErr := s.lock.release() + s.closeErr = errors.Join(saveErr, relErr) + }) + return s.closeErr +} + +// runCheckpoints calls fn on every interval tick until ctx is cancelled. +func runCheckpoints(ctx context.Context, interval time.Duration, fn func()) { + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + fn() + } + } +} diff --git a/internal/profile/session_test.go b/internal/profile/session_test.go new file mode 100644 index 0000000..225dbc4 --- /dev/null +++ b/internal/profile/session_test.go @@ -0,0 +1,198 @@ +package profile + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/glim-sh/cuttle/internal/cdp" +) + +const testCDPBase = "http://127.0.0.1:9222" + +var errInjectBoom = errors.New("inject boom") + +func TestRunCheckpointsTicksThenStops(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + ticks := make(chan struct{}, 16) + done := make(chan struct{}) + go func() { + runCheckpoints(ctx, 5*time.Millisecond, func() { ticks <- struct{}{} }) + close(done) + }() + for i := range 3 { + select { + case <-ticks: + case <-time.After(2 * time.Second): + t.Fatalf("checkpoint tick %d did not fire", i) + } + } + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("runCheckpoints did not stop after cancel") + } +} + +func TestCheckoutInjectsAndCheckinSaves(t *testing.T) { + t.Setenv("XDG_DATA_HOME", t.TempDir()) + const name = "linkedin" + + initial := &cdp.StorageState{ + Cookies: []cdp.Cookie{{Name: "old", Value: "1", Domain: exampleDomain, Path: "/", Expires: -1}}, + } + if err := saveState(DataDir(name), initial); err != nil { + t.Fatalf("seed profile: %v", err) + } + + var injected atomic.Value + inject := func(_ context.Context, _, seed string, st *cdp.StorageState) error { + if seed != name { + t.Errorf("inject seed=%q want %q", seed, name) + } + injected.Store(st) + return nil + } + updated := &cdp.StorageState{ + Cookies: []cdp.Cookie{{Name: "new", Value: "2", Domain: exampleDomain, Path: "/", Expires: -1}}, + } + extract := func(_ context.Context, _, _ string, _ []string) (*cdp.StorageState, []string, error) { + return updated, nil, nil + } + + s, err := checkoutSession(t.Context(), Options{Name: name, CDPBase: testCDPBase, Interval: time.Hour}, inject, extract) + if err != nil { + t.Fatalf("checkout: %v", err) + } + + got, ok := injected.Load().(*cdp.StorageState) + if !ok || got == nil || len(got.Cookies) != 1 || got.Cookies[0].Name != "old" { + t.Fatalf("inject received %+v, want the seeded state", got) + } + + if cerr := s.Close(); cerr != nil { + t.Fatalf("close: %v", cerr) + } + + final, err := loadState(DataDir(name)) + if err != nil { + t.Fatalf("reload: %v", err) + } + if len(final.Cookies) != 1 || final.Cookies[0].Name != "new" { + t.Fatalf("checkin wrote %+v, want the extracted state", final.Cookies) + } + + if _, serr := os.Stat(filepath.Join(DataDir(name), lockName)); !errors.Is(serr, os.ErrNotExist) { + t.Fatalf("lock not released: %v", serr) + } +} + +func TestCheckinCarriesForwardFailedOriginLocalStorage(t *testing.T) { + t.Setenv("XDG_DATA_HOME", t.TempDir()) + const name = "linkedin" + const failedOrigin = "https://example.com" + const emptiedOrigin = "https://cleared.example" + + seeded := &cdp.StorageState{ + Origins: []cdp.Origin{ + {Origin: failedOrigin, LocalStorage: []cdp.LocalStorageItem{{Name: "token", Value: "keep-me"}}}, + {Origin: emptiedOrigin, LocalStorage: []cdp.LocalStorageItem{{Name: "token", Value: "gone"}}}, + }, + } + if err := saveState(DataDir(name), seeded); err != nil { + t.Fatalf("seed profile: %v", err) + } + + inject := func(context.Context, string, string, *cdp.StorageState) error { return nil } + // The extract loaded emptiedOrigin (now genuinely empty, so absent from the + // result) and failed to load failedOrigin (reported in failed). + extract := func(context.Context, string, string, []string) (*cdp.StorageState, []string, error) { + return &cdp.StorageState{}, []string{failedOrigin}, nil + } + + s, err := checkoutSession(t.Context(), Options{Name: name, CDPBase: testCDPBase, Interval: time.Hour}, inject, extract) + if err != nil { + t.Fatalf("checkout: %v", err) + } + if cerr := s.Close(); cerr != nil { + t.Fatalf("close: %v", cerr) + } + + final, err := loadState(DataDir(name)) + if err != nil { + t.Fatalf("reload: %v", err) + } + byOrigin := map[string][]cdp.LocalStorageItem{} + for _, o := range final.Origins { + byOrigin[o.Origin] = o.LocalStorage + } + if ls := byOrigin[failedOrigin]; len(ls) != 1 || ls[0].Value != "keep-me" { + t.Errorf("failed origin localStorage = %+v, want it carried forward (keep-me)", ls) + } + if _, ok := byOrigin[emptiedOrigin]; ok { + t.Errorf("emptied origin should be dropped (real clear), got %+v", byOrigin[emptiedOrigin]) + } +} + +func TestCheckoutRejectsSecondSession(t *testing.T) { + t.Setenv("XDG_DATA_HOME", t.TempDir()) + nop := func(context.Context, string, string, *cdp.StorageState) error { return nil } + ext := func(context.Context, string, string, []string) (*cdp.StorageState, []string, error) { + return &cdp.StorageState{}, nil, nil + } + opts := Options{Name: "x", CDPBase: testCDPBase, Interval: time.Hour} + + s1, err := checkoutSession(t.Context(), opts, nop, ext) + if err != nil { + t.Fatalf("first checkout: %v", err) + } + if _, serr := checkoutSession(t.Context(), opts, nop, ext); !errors.Is(serr, errCheckedOut) { + t.Fatalf("second checkout: want errCheckedOut, got %v", serr) + } + if cerr := s1.Close(); cerr != nil { + t.Fatalf("close: %v", cerr) + } +} + +func TestCheckoutInjectFailureReleasesLock(t *testing.T) { + t.Setenv("XDG_DATA_HOME", t.TempDir()) + inject := func(context.Context, string, string, *cdp.StorageState) error { return errInjectBoom } + ext := func(context.Context, string, string, []string) (*cdp.StorageState, []string, error) { + return &cdp.StorageState{}, nil, nil + } + opts := Options{Name: "x", CDPBase: testCDPBase} + if _, err := checkoutSession(t.Context(), opts, inject, ext); !errors.Is(err, errInjectBoom) { + t.Fatalf("want inject error, got %v", err) + } + if _, err := os.Stat(filepath.Join(DataDir("x"), lockName)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("lock should be released after inject failure: %v", err) + } +} + +func TestRemoteSessionIsInert(t *testing.T) { + t.Setenv("XDG_DATA_HOME", t.TempDir()) + inject := func(context.Context, string, string, *cdp.StorageState) error { + t.Error("remote session must not inject") + return nil + } + ext := func(context.Context, string, string, []string) (*cdp.StorageState, []string, error) { + t.Error("remote session must not extract") + return nil, nil, nil + } + s, err := checkoutSession(t.Context(), Options{Name: "auto", Remote: true}, inject, ext) + if err != nil { + t.Fatalf("remote checkout: %v", err) + } + if cerr := s.Close(); cerr != nil { + t.Fatalf("close: %v", cerr) + } + if _, serr := os.Stat(filepath.Join(DataDir("auto"), lockName)); !errors.Is(serr, os.ErrNotExist) { + t.Fatalf("remote session should not create a lock: %v", serr) + } +} diff --git a/internal/serve/http.go b/internal/serve/http.go new file mode 100644 index 0000000..0140878 --- /dev/null +++ b/internal/serve/http.go @@ -0,0 +1,329 @@ +package serve + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/netip" + "net/url" + "strconv" + "strings" + "time" +) + +// Repeated string literals shared across the serve package. +const ( + schemeHTTP = "http" + schemeHTTPS = "https" + keyLocale = "locale" + keyProxy = "proxy" + keyTimezone = "timezone" + keyError = "error" + msgChromeFailed = "Chrome failed to start" +) + +// specialParams are handled explicitly; any other query param becomes a generic +// --fingerprint-{key}={val} passthrough. +var specialParams = map[string]struct{}{ + "fingerprint": {}, keyProxy: {}, "geoip": {}, keyLocale: {}, keyTimezone: {}, +} + +var trustedWSOrigins = map[string]struct{}{ + "devtools://devtools": {}, + "chrome-devtools://devtools": {}, +} + +// multiplexer holds the shared pool and the advertised port for URL rewrites. +type multiplexer struct { + pool *chromePool + port int +} + +func (m *multiplexer) routes() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("GET /{$}", m.handleRoot) + for _, p := range []string{"GET /json/version", "GET /json/version/"} { + mux.HandleFunc(p, m.handleJSONVersion) + } + for _, p := range []string{"GET /json/list", "GET /json/list/", "GET /json", "GET /json/"} { + mux.HandleFunc(p, m.handleJSONList) + } + mux.HandleFunc("GET /fingerprint/{seed}/devtools/{path...}", m.handleWSSeed) + mux.HandleFunc("GET /devtools/{path...}", m.handleWSDefault) + return mux +} + +// parseConnectionParams parses a raw query string into a connection request, +// mirroring parse_qs(keep_blank_values=False): blank values are dropped and the +// first value per key wins. Unknown params map to --fingerprint-{key}={val}. +func parseConnectionParams(raw string) connectRequest { + req := connectRequest{} + seen := map[string]struct{}{} + for pair := range strings.SplitSeq(raw, "&") { + if pair == "" { + continue + } + rawKey, rawVal, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil { + continue + } + val, err := url.QueryUnescape(rawVal) + if err != nil || val == "" { + continue + } + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + switch key { + case "fingerprint": + req.seed = val + case keyTimezone: + req.timezone = val + case keyLocale: + req.locale = val + case keyProxy: + req.proxy = val + case "geoip": + l := strings.ToLower(val) + req.geoip = l == "true" || l == "1" || l == "yes" + default: + if _, special := specialParams[key]; !special { + req.extraArgs = append(req.extraArgs, "--fingerprint-"+key+"="+val) + } + } + } + return req +} + +func (m *multiplexer) handleRoot(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, m.pool.status()) +} + +func (m *multiplexer) handleJSONVersion(w http.ResponseWriter, r *http.Request) { + params := parseConnectionParams(r.URL.RawQuery) + cp, err := m.pool.getOrLaunch(r.Context(), params) + if err != nil { + writeLaunchError(w, err) + return + } + + var data map[string]any + if err := fetchCDP(r.Context(), cp.cdpPort, "/json/version", &data); err != nil { + logError("failed to reach Chrome CDP (port %d): %v", cp.cdpPort, err) + writeJSON(w, http.StatusBadGateway, map[string]any{keyError: "CDP endpoint unreachable"}) + return + } + + host := externalHost(r, m.port) + scheme := wsScheme(r) + wsPath := "devtools/browser" + if params.seed != "" { + wsPath = "fingerprint/" + params.seed + "/devtools/browser" + } + guid := "" + if orig, ok := data["webSocketDebuggerUrl"].(string); ok && strings.Contains(orig, "/devtools/") { + guid = orig[strings.LastIndex(orig, "/")+1:] + } + data["webSocketDebuggerUrl"] = scheme + "://" + host + "/" + wsPath + "/" + guid + writeJSON(w, http.StatusOK, data) +} + +func (m *multiplexer) handleJSONList(w http.ResponseWriter, r *http.Request) { + params := parseConnectionParams(r.URL.RawQuery) + cp, err := m.pool.getOrLaunch(r.Context(), params) + if err != nil { + writeLaunchError(w, err) + return + } + + var data []map[string]any + if err := fetchCDP(r.Context(), cp.cdpPort, "/json/list", &data); err != nil { + logError("failed to reach Chrome CDP (port %d): %v", cp.cdpPort, err) + writeJSON(w, http.StatusBadGateway, map[string]any{keyError: "CDP endpoint unreachable"}) + return + } + + host := externalHost(r, m.port) + scheme := wsScheme(r) + for _, entry := range data { + orig, ok := entry["webSocketDebuggerUrl"].(string) + if !ok { + continue + } + tail := orig[strings.LastIndex(orig, "/devtools/")+len("/devtools/"):] + if params.seed != "" { + entry["webSocketDebuggerUrl"] = scheme + "://" + host + "/fingerprint/" + params.seed + "/devtools/" + tail + } else { + entry["webSocketDebuggerUrl"] = scheme + "://" + host + "/devtools/" + tail + } + } + writeJSON(w, http.StatusOK, data) +} + +func writeLaunchError(w http.ResponseWriter, err error) { + var le *launchError + if errors.As(err, &le) { + writeJSON(w, le.status, map[string]any{keyError: le.msg}) + return + } + writeJSON(w, http.StatusBadGateway, map[string]any{keyError: msgChromeFailed}) +} + +func fetchCDP(ctx context.Context, port int, path string, out any) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + endpoint := "http://127.0.0.1:" + strconv.Itoa(port) + path + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return err //nolint:wrapcheck + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err //nolint:wrapcheck + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return err //nolint:wrapcheck + } + return json.Unmarshal(body, out) //nolint:wrapcheck +} + +// externalHost is the public host used in rewritten CDP WebSocket URLs, so they +// are correct behind kubectl port-forward / ssh -L. X-Forwarded-Host wins, then +// the request Host header, then a localhost fallback. +func externalHost(r *http.Request, port int) string { + if fwd := r.Header.Get("X-Forwarded-Host"); fwd != "" { + if h := strings.TrimSpace(strings.SplitN(fwd, ",", 2)[0]); h != "" { + return h + } + } + if r.Host != "" { + return r.Host + } + return "localhost:" + strconv.Itoa(port) +} + +func wsScheme(r *http.Request) string { + if requestScheme(r) == schemeHTTPS { + return "wss" + } + return "ws" +} + +func requestScheme(r *http.Request) string { + proto := r.Header.Get("X-Forwarded-Proto") + if proto == "" { + if r.TLS != nil { + return schemeHTTPS + } + return schemeHTTP + } + return strings.ToLower(strings.TrimSpace(strings.SplitN(proto, ",", 2)[0])) +} + +// --------------------------------------------------------------------------- +// WebSocket Origin allow-list +// --------------------------------------------------------------------------- + +// rejectUntrustedOrigin blocks browser-origin WebSocket upgrades that would +// expose local CDP, while still allowing non-browser clients (which omit Origin) +// and same-origin loopback clients (kubectl port-forward / ssh -L). Returns true +// when it wrote a 403. +func (m *multiplexer) rejectUntrustedOrigin(w http.ResponseWriter, r *http.Request) bool { + origin, present := r.Header["Origin"] + value := "" + if present && len(origin) > 0 { + value = origin[0] + } + if originIsAllowed(value, present, r.Host, requestScheme(r)) { + return false + } + logWarn("rejected CDP WebSocket from untrusted Origin %q for Host %q", value, r.Host) + http.Error(w, "Forbidden: untrusted WebSocket origin", http.StatusForbidden) + return true +} + +func originIsAllowed(origin string, present bool, host, scheme string) bool { + if !present { + // Playwright/Puppeteer and other non-browser CDP clients omit Origin. + return true + } + origin = strings.TrimSpace(origin) + if origin == "" || strings.EqualFold(origin, "null") { + return false + } + if _, ok := trustedWSOrigins[origin]; ok { + return true + } + u, err := url.Parse(origin) + if err != nil { + return false + } + if u.Scheme != schemeHTTP && u.Scheme != schemeHTTPS { + return false + } + if u.Path != "" || u.RawQuery != "" || u.Fragment != "" || u.User != nil { + return false + } + originDefaultPort := 80 + if u.Scheme == schemeHTTPS { + originDefaultPort = 443 + } + rs := strings.ToLower(strings.TrimSpace(strings.SplitN(scheme, ",", 2)[0])) + requestDefaultPort := 80 + if rs == schemeHTTPS || rs == "wss" { + requestDefaultPort = 443 + } + originHost, originPort, ok := hostPortFromNetloc(u.Host, originDefaultPort) + if !ok { + return false + } + requestHost, requestPort, ok := hostPortFromNetloc(host, requestDefaultPort) + if !ok { + return false + } + if !isLoopbackHost(requestHost) { + return false + } + return originHost == requestHost && originPort == requestPort +} + +func hostPortFromNetloc(netloc string, defaultPort int) (string, int, bool) { + if strings.Contains(netloc, ",") { + return "", 0, false + } + u, err := url.Parse("//" + strings.TrimSpace(netloc)) + if err != nil { + return "", 0, false + } + if u.Hostname() == "" || u.User != nil || strings.HasSuffix(u.Host, ":") || + u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return "", 0, false + } + port := defaultPort + if ps := u.Port(); ps != "" { + p, err := strconv.Atoi(ps) + if err != nil { + return "", 0, false + } + port = p + } + return strings.ToLower(u.Hostname()), port, true +} + +func isLoopbackHost(hostname string) bool { + hostname = strings.ToLower(strings.TrimRight(strings.Trim(hostname, "[]"), ".")) + if hostname == "localhost" { + return true + } + addr, err := netip.ParseAddr(hostname) + if err != nil { + return false + } + return addr.IsLoopback() +} diff --git a/internal/serve/http_test.go b/internal/serve/http_test.go new file mode 100644 index 0000000..b7616fc --- /dev/null +++ b/internal/serve/http_test.go @@ -0,0 +1,104 @@ +package serve + +import ( + "slices" + "testing" +) + +func TestParseConnectionParams(t *testing.T) { + t.Parallel() + tests := []struct { + name string + query string + want connectRequest + }{ + { + name: "all recognized params", + query: "fingerprint=seed1&timezone=America/New_York&locale=en-US&proxy=http://p:1&geoip=true", + want: connectRequest{ + seed: "seed1", timezone: "America/New_York", locale: "en-US", + proxy: "http://p:1", geoip: true, + }, + }, + { + name: "generic param becomes fingerprint flag", + query: "fingerprint=s&platform=windows&webrtc-ip=auto", + want: connectRequest{ + seed: "s", + extraArgs: []string{"--fingerprint-platform=windows", "--fingerprint-webrtc-ip=auto"}, + }, + }, + { + name: "blank values dropped", + query: "fingerprint=&timezone=&locale=en-GB", + want: connectRequest{locale: "en-GB"}, + }, + { + name: "geoip falsey", + query: "fingerprint=s&geoip=no", + want: connectRequest{seed: "s", geoip: false}, + }, + { + name: "first value wins on duplicate key", + query: "fingerprint=first&fingerprint=second", + want: connectRequest{seed: "first"}, + }, + { + name: "url-encoded value", + query: "proxy=http%3A%2F%2Fu%3Ap%40h%3A8080", + want: connectRequest{proxy: "http://u:p@h:8080"}, + }, + { + name: "empty query", + query: "", + want: connectRequest{}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := parseConnectionParams(tc.query) + if got.seed != tc.want.seed || got.timezone != tc.want.timezone || + got.locale != tc.want.locale || got.proxy != tc.want.proxy || got.geoip != tc.want.geoip { + t.Errorf("scalar mismatch: got %+v want %+v", got, tc.want) + } + if !slices.Equal(got.extraArgs, tc.want.extraArgs) { + t.Errorf("extraArgs got %v want %v", got.extraArgs, tc.want.extraArgs) + } + }) + } +} + +func TestOriginIsAllowed(t *testing.T) { + t.Parallel() + tests := []struct { + name string + origin string + present bool + host string + scheme string + want bool + }{ + {name: "absent origin allowed (non-browser client)", present: false, host: "127.0.0.1:9222", scheme: "http", want: true}, + {name: "empty origin rejected", origin: "", present: true, host: "127.0.0.1:9222", scheme: "http", want: false}, + {name: "null origin rejected", origin: "null", present: true, host: "127.0.0.1:9222", scheme: "http", want: false}, + {name: "devtools trusted", origin: "devtools://devtools", present: true, host: "example:9222", scheme: "http", want: true}, + {name: "loopback same-origin allowed (port-forward)", origin: "http://127.0.0.1:9222", present: true, host: "127.0.0.1:9222", scheme: "http", want: true}, + {name: "localhost same-origin allowed", origin: "http://localhost:9222", present: true, host: "localhost:9222", scheme: "http", want: true}, + {name: "port mismatch rejected", origin: "http://127.0.0.1:9223", present: true, host: "127.0.0.1:9222", scheme: "http", want: false}, + {name: "non-loopback host rejected", origin: "http://evil.example", present: true, host: "evil.example", scheme: "http", want: false}, + {name: "cross-origin against loopback rejected", origin: "http://evil.example", present: true, host: "127.0.0.1:9222", scheme: "http", want: false}, + {name: "origin with path rejected", origin: "http://127.0.0.1:9222/x", present: true, host: "127.0.0.1:9222", scheme: "http", want: false}, + {name: "origin with userinfo rejected", origin: "http://u@127.0.0.1:9222", present: true, host: "127.0.0.1:9222", scheme: "http", want: false}, + {name: "bad scheme rejected", origin: "ftp://127.0.0.1", present: true, host: "127.0.0.1", scheme: "http", want: false}, + {name: "default ports match (http 80)", origin: "http://localhost", present: true, host: "localhost:80", scheme: "http", want: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := originIsAllowed(tc.origin, tc.present, tc.host, tc.scheme); got != tc.want { + t.Errorf("originIsAllowed(%q,%v,%q,%q)=%v want %v", tc.origin, tc.present, tc.host, tc.scheme, got, tc.want) + } + }) + } +} diff --git a/internal/serve/pool.go b/internal/serve/pool.go new file mode 100644 index 0000000..f6f77b0 --- /dev/null +++ b/internal/serve/pool.go @@ -0,0 +1,659 @@ +package serve + +import ( + "context" + "encoding/json" + "errors" + "io" + "math/rand/v2" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/glim-sh/cuttle/internal/fingerprint" +) + +// launchError carries an HTTP status for a get-or-launch failure so handlers can +// translate it into a CDP-style JSON error before any WebSocket upgrade. +type launchError struct { + status int + msg string +} + +func (e *launchError) Error() string { return e.msg } + +// processHandle abstracts a spawned Chrome so the pool can be tested without a +// real browser. +type processHandle interface { + running() bool + signalTerm() error + kill() error + wait(timeout time.Duration) bool + pid() int +} + +// launcher is the injectable seam that allocates a CDP port, starts the process, +// and waits for its CDP endpoint to answer. The default wires os/exec + a real +// port probe; tests substitute a fake CDP server. +type launcher struct { + allocPort func() (int, error) + start func(binary string, args []string) (processHandle, error) + waitReady func(ctx context.Context, port int) bool +} + +type chromeInstance struct { + seed string + process processHandle + cdpPort int + userDataDir string + timezone string + locale string + proxy string +} + +type chromePool struct { + binary string + globalArgs []string + headless bool + dataDir string + defaultSeed string + defaultLocale string + defaultTimezone string + defaultProxy string + idleTimeout time.Duration + keepProfile bool + ephemeral bool + launch launcher + geo fingerprint.GeoResolver + + mu sync.Mutex + processes map[string]*chromeInstance + seedLocks map[string]*sync.Mutex + conns map[string]int + idleTimers map[string]*time.Timer +} + +func newChromePool(cfg serveConfig, binary string, globalArgs []string, l launcher, geo fingerprint.GeoResolver) *chromePool { + return &chromePool{ + binary: binary, + globalArgs: globalArgs, + headless: cfg.headless, + dataDir: cfg.dataDir, + defaultSeed: cfg.defaultSeed, + defaultLocale: cfg.defaultLocale, + defaultTimezone: cfg.defaultTimezone, + defaultProxy: cfg.proxy, + idleTimeout: cfg.idleTimeout, + keepProfile: cfg.keepProfile, + ephemeral: cfg.ephemeral, + launch: l, + geo: geo, + processes: map[string]*chromeInstance{}, + seedLocks: map[string]*sync.Mutex{}, + conns: map[string]int{}, + idleTimers: map[string]*time.Timer{}, + } +} + +func (p *chromePool) seedLock(key string) *sync.Mutex { + p.mu.Lock() + defer p.mu.Unlock() + l := p.seedLocks[key] + if l == nil { + l = &sync.Mutex{} + p.seedLocks[key] = l + } + return l +} + +// connect increments a seed's connection refcount and cancels any pending idle +// reap. +func (p *chromePool) connect(seedKey string) { + p.mu.Lock() + defer p.mu.Unlock() + p.cancelIdleLocked(seedKey) + p.conns[seedKey]++ +} + +// disconnect decrements a seed's refcount and schedules an idle reap when it +// reaches zero. +func (p *chromePool) disconnect(seedKey string) { + p.mu.Lock() + defer p.mu.Unlock() + p.conns[seedKey]-- + if p.conns[seedKey] <= 0 { + delete(p.conns, seedKey) + p.scheduleIdleLocked(seedKey) + } +} + +func (p *chromePool) cancelIdleLocked(seedKey string) { + if t := p.idleTimers[seedKey]; t != nil { + t.Stop() + delete(p.idleTimers, seedKey) + } +} + +// scheduleIdleLocked arms an idle reap. Reaping is disabled when idleTimeout <= 0 +// (remote browsers never reap); local runs pass a positive timeout. +func (p *chromePool) scheduleIdleLocked(seedKey string) { + if p.idleTimeout <= 0 { + return + } + if _, ok := p.processes[seedKey]; !ok { + return + } + p.cancelIdleLocked(seedKey) + p.idleTimers[seedKey] = time.AfterFunc(p.idleTimeout, func() { p.idleReap(seedKey) }) +} + +func (p *chromePool) idleReap(seedKey string) { + p.mu.Lock() + if p.conns[seedKey] > 0 { + p.mu.Unlock() + return + } + inst := p.processes[seedKey] + if inst == nil { + p.mu.Unlock() + return + } + delete(p.processes, seedKey) + delete(p.idleTimers, seedKey) + delete(p.conns, seedKey) + delete(p.seedLocks, seedKey) + p.mu.Unlock() + + logInfo("cleaning up idle Chrome process (seed=%s)", seedKey) + p.terminate(inst) +} + +// connectRequest carries the per-connection parameters resolved from the query +// string (and server defaults). +type connectRequest struct { + seed string + extraArgs []string + timezone string + locale string + proxy string + geoip bool +} + +// getOrLaunch returns the running Chrome for a seed, launching it on first use. +// A missing seed maps to the shared "__default__" process with a random +// fingerprint. First-launch wins: later params for a live seed are ignored. +func (p *chromePool) getOrLaunch(ctx context.Context, req connectRequest) (*chromeInstance, error) { + seed := req.seed + if seed == "" && p.defaultSeed != "" { + seed = p.defaultSeed + } + locale := req.locale + if locale == "" { + locale = p.defaultLocale + } + timezone := req.timezone + if timezone == "" { + timezone = p.defaultTimezone + } + proxy := req.proxy + if proxy == "" { + proxy = p.defaultProxy + } + + var seedKey, actualSeed string + if seed == "" { + seedKey = reservedSeed + actualSeed = strconv.Itoa(randSeed()) + } else { + if !validSeed(seed) { + return nil, &launchError{status: http.StatusBadRequest, msg: "Invalid fingerprint seed"} + } + seedKey = seed + actualSeed = seed + } + + lock := p.seedLock(seedKey) + lock.Lock() + defer lock.Unlock() + + p.mu.Lock() + existing := p.processes[seedKey] + pending := p.idleTimers[seedKey] != nil + p.mu.Unlock() + + if existing != nil && existing.process.running() { + if pending { + p.mu.Lock() + p.scheduleIdleLocked(seedKey) + p.mu.Unlock() + } + if len(req.extraArgs) > 0 || timezone != "" || locale != "" || proxy != "" || req.geoip { + logWarn("seed %s already running (port %d, tz=%s, locale=%s, proxy=%s) - ignoring new params (first-launch wins)", + seedKey, existing.cdpPort, existing.timezone, existing.locale, existing.proxy) + } + return existing, nil + } + if existing != nil { + p.removeProcess(seedKey) + p.terminate(existing) + } + + var exitIP string + if req.geoip && proxy != "" { + timezone, locale, exitIP = p.resolveGeo(proxy, timezone, locale) + } + + fpExtra := []string{"--fingerprint=" + actualSeed} + fpExtra = append(fpExtra, req.extraArgs...) + if proxy != "" { + // Fork binaries reject inline creds on --proxy-server; strip them here + // and answer the proxy 407 over CDP (see wsproxy). geoip above still uses + // the credentialed proxy; the full proxy is stored on the instance so the + // ws proxy can recover the credentials. + stripped, _, _ := fingerprint.SplitProxyAuth(proxy) + fpExtra = append(fpExtra, "--proxy-server="+fingerprint.NormalizeSocksStringURL(stripped)) + } + + // resolveGeo above already resolved the exit IP over the network; reuse it for + // --fingerprint-webrtc-ip=auto instead of letting ResolveWebRTCArgs hit the + // echo services a second time on the launch path. + webrtcResolver := p.exitIPForWebRTC + if exitIP != "" { + webrtcResolver = func(string) string { return exitIP } + } + fpExtra = fingerprint.ResolveWebRTCArgs(fpExtra, proxy, webrtcResolver) + if exitIP != "" && !slices.ContainsFunc(fpExtra, func(a string) bool { + return strings.HasPrefix(a, "--fingerprint-webrtc-ip") + }) { + fpExtra = append(fpExtra, "--fingerprint-webrtc-ip="+exitIP) + } + + fpExtra = append(fpExtra, fingerprint.ForkParityArgs(locale, proxy)...) + + chromeArgs := fingerprint.BuildArgs(fingerprint.BuildArgsInput{ + StealthArgs: true, + ExtraArgs: fpExtra, + Timezone: timezone, + Locale: locale, + Headless: p.headless, + }) + + inst, err := p.spawn(ctx, seedKey, actualSeed, chromeArgs, timezone, locale, proxy) + if err != nil { + return nil, err + } + + p.mu.Lock() + p.processes[seedKey] = inst + p.mu.Unlock() + return inst, nil +} + +func (p *chromePool) spawn(ctx context.Context, seedKey, actualSeed string, chromeArgs []string, timezone, locale, proxy string) (*chromeInstance, error) { + userDataDir, err := p.profileDir(seedKey) + if err != nil { + logError("failed to create profile dir for seed=%s: %v", seedKey, err) + return nil, &launchError{status: http.StatusBadGateway, msg: msgChromeFailed} + } + seedProfileDefaults(userDataDir) + + port, err := p.launch.allocPort() + if err != nil { + p.safeRemoveTree(userDataDir) + return nil, &launchError{status: http.StatusBadGateway, msg: msgChromeFailed} + } + + fullArgs := slices.Clone(baseChromeArgs) + fullArgs = append(fullArgs, chromeArgs...) + fullArgs = append(fullArgs, p.globalArgs...) + fullArgs = append( + fullArgs, + "--remote-debugging-port="+strconv.Itoa(port), + "--remote-debugging-address=127.0.0.1", + "--user-data-dir="+userDataDir, + ) + + logInfo("launching Chrome (seed=%s, port=%d)", actualSeed, port) + proc, err := p.launch.start(p.binary, fullArgs) + if err != nil { + p.safeRemoveTree(userDataDir) + return nil, &launchError{status: http.StatusBadGateway, msg: msgChromeFailed} + } + + if !p.launch.waitReady(ctx, port) { + _ = proc.kill() + proc.wait(terminateGrace) + p.safeRemoveTree(userDataDir) + return nil, &launchError{status: http.StatusBadGateway, msg: msgChromeFailed} + } + + logInfo("Chrome ready (seed=%s, port=%d, pid=%d)", actualSeed, port, proc.pid()) + return &chromeInstance{ + seed: actualSeed, + process: proc, + cdpPort: port, + userDataDir: userDataDir, + timezone: timezone, + locale: locale, + proxy: proxy, + }, nil +} + +// profileDir returns the seed's user_data_dir. When ephemeral, it is a fresh +// scratch dir under dataDir (nothing persists across sessions); otherwise it is +// the stable per-seed path. +func (p *chromePool) profileDir(seedKey string) (string, error) { + if err := os.MkdirAll(p.dataDir, 0o700); err != nil { + return "", err //nolint:wrapcheck + } + if p.ephemeral { + dir, err := os.MkdirTemp(p.dataDir, seedKey+"-") + if err != nil { + return "", err //nolint:wrapcheck + } + return dir, nil + } + dir := filepath.Join(p.dataDir, seedKey) + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", err //nolint:wrapcheck + } + return dir, nil +} + +func (p *chromePool) removeProcess(seedKey string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.processes, seedKey) + p.cancelIdleLocked(seedKey) + delete(p.conns, seedKey) +} + +// terminate stops a Chrome (SIGTERM, then SIGKILL after the grace period) and +// removes its profile dir. It blocks and must be called without p.mu held. +func (p *chromePool) terminate(inst *chromeInstance) { + if inst.process.running() { + _ = inst.process.signalTerm() + if !inst.process.wait(terminateGrace) { + _ = inst.process.kill() + } + } + p.safeRemoveTree(inst.userDataDir) +} + +// safeRemoveTree deletes a profile dir, refusing any path outside dataDir. An +// ephemeral profile is always removed; otherwise --keep-profile preserves it. +func (p *chromePool) safeRemoveTree(path string) { + if p.keepProfile && !p.ephemeral { + logInfo("keep-profile: preserving %s", path) + return + } + resolved, err := filepath.Abs(path) + if err != nil { + return + } + dataResolved, err := filepath.Abs(p.dataDir) + if err != nil { + return + } + if resolved == dataResolved || !withinDir(dataResolved, resolved) { + logError("refusing to delete path outside data_dir: %s", resolved) + return + } + _ = os.RemoveAll(path) +} + +func withinDir(dir, path string) bool { + rel, err := filepath.Rel(dir, path) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// shutdown terminates every Chrome process on daemon exit. +func (p *chromePool) shutdown() { + p.mu.Lock() + for key, t := range p.idleTimers { + t.Stop() + delete(p.idleTimers, key) + } + insts := make([]*chromeInstance, 0, len(p.processes)) + keys := make([]string, 0, len(p.processes)) + for key, inst := range p.processes { + keys = append(keys, key) + insts = append(insts, inst) + } + for _, key := range keys { + delete(p.processes, key) + delete(p.conns, key) + } + p.mu.Unlock() + + for _, inst := range insts { + p.terminate(inst) + } + logInfo("all Chrome processes terminated") +} + +// status reports the live process table for the health-check endpoint. +func (p *chromePool) status() map[string]any { + p.mu.Lock() + defer p.mu.Unlock() + procs := map[string]any{} + for key, inst := range p.processes { + if !inst.process.running() { + continue + } + procs[key] = map[string]any{ + "pid": inst.process.pid(), + "port": inst.cdpPort, + "seed": inst.seed, + "connections": p.conns[key], + "idle_cleanup_pending": p.idleTimers[key] != nil, + "timezone": inst.timezone, + "locale": inst.locale, + "proxy": inst.proxy, + } + } + return map[string]any{ + "status": "ok", + "active": len(procs), + "idle_timeout": p.idleTimeout.Seconds(), + "processes": procs, + } +} + +// resolveGeo mirrors maybe_resolve_geoip: when both tz and locale are explicit, +// only the exit IP is resolved (for WebRTC); otherwise tz/locale are filled from +// the egress geo. The credentialed proxy is used so the egress IP is the proxy's +// exit IP. +func (p *chromePool) resolveGeo(proxy, timezone, locale string) (string, string, string) { + proxyURL := fingerprint.EnsureProxyScheme(proxy) + if timezone != "" && locale != "" { + exitIP := "" + if p.geo.ExitIP != nil { + if ip, err := p.geo.ExitIP(proxyURL); err == nil { + exitIP = ip + } + } + return timezone, locale, exitIP + } + geoTZ, geoLocale, exitIP := p.geo.ResolveProxyGeoWithIP(proxyURL) + if timezone == "" { + timezone = geoTZ + } + if locale == "" { + locale = geoLocale + } + return timezone, locale, exitIP +} + +func (p *chromePool) exitIPForWebRTC(proxyURL string) string { + if p.geo.ExitIP == nil { + return "" + } + ip, err := p.geo.ExitIP(proxyURL) + if err != nil { + return "" + } + return ip +} + +// seedProfileDefaults writes DuckDuckGo as the default search on a brand-new +// profile (matching the upstream seeding). Chrome owns the file afterward; tab +// restore is handled by clean shutdown, not by forging flags here. +func seedProfileDefaults(userDataDir string) { + defaultDir := filepath.Join(userDataDir, "Default") + prefsPath := filepath.Join(defaultDir, "Preferences") + if _, err := os.Stat(prefsPath); err == nil { + return + } + if err := os.MkdirAll(defaultDir, 0o700); err != nil { + return + } + prefs := map[string]any{ + "default_search_provider_data": map[string]any{ + "template_url_data": map[string]any{ + "keyword": "duckduckgo.com", + "short_name": "DuckDuckGo", + "url": "https://duckduckgo.com/?q={searchTerms}", + "suggestions_url": "https://duckduckgo.com/ac/?q={searchTerms}&type=list", + "favicon_url": "https://duckduckgo.com/favicon.ico", + }, + }, + "default_search_provider": map[string]any{"enabled": true}, + } + data, err := json.Marshal(prefs) + if err != nil { + return + } + _ = os.WriteFile(prefsPath, data, 0o600) +} + +func randSeed() int { + // A fingerprint seed, not a security token; math/rand mirrors the oracle. + return rand.IntN(90000) + 10000 //nolint:gosec // non-cryptographic seed +} + +// --------------------------------------------------------------------------- +// Default launcher (os/exec) +// --------------------------------------------------------------------------- + +func defaultLauncher() launcher { + return launcher{ + allocPort: newSequentialPortAllocator(), + start: startChrome, + waitReady: waitForCDP, + } +} + +var errNoFreePorts = errors.New("no free ports available for Chrome CDP") + +func newSequentialPortAllocator() func() (int, error) { + var mu sync.Mutex + next := basePort + return func() (int, error) { + mu.Lock() + defer mu.Unlock() + var lc net.ListenConfig + for range 100 { + port := next + next++ + l, err := lc.Listen(context.Background(), "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + continue + } + _ = l.Close() + return port, nil + } + return 0, errNoFreePorts + } +} + +func startChrome(binary string, args []string) (processHandle, error) { + // context.Background, not any request context: the pool owns Chrome's + // lifecycle (signalTerm/kill), so the process must outlive the HTTP request + // that launched it. + cmd := exec.CommandContext(context.Background(), binary, args...) + cmd.Stdout = nil + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + return nil, err //nolint:wrapcheck + } + h := &osProcess{cmd: cmd, done: make(chan struct{})} + go func() { + _ = cmd.Wait() + h.mu.Lock() + h.exited = true + h.mu.Unlock() + close(h.done) + }() + return h, nil +} + +func waitForCDP(ctx context.Context, port int) bool { + deadline := time.Now().Add(10 * time.Second) + delay := 100 * time.Millisecond + endpoint := "http://127.0.0.1:" + strconv.Itoa(port) + "/json/version" + client := &http.Client{Timeout: time.Second} + for time.Now().Before(deadline) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err == nil { + resp, err := client.Do(req) + if err == nil { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16)) + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return true + } + } + } + select { + case <-ctx.Done(): + return false + case <-time.After(delay): + } + delay = min(delay*2, time.Second) + } + return false +} + +type osProcess struct { + cmd *exec.Cmd + mu sync.Mutex + done chan struct{} + exited bool +} + +func (o *osProcess) running() bool { + o.mu.Lock() + defer o.mu.Unlock() + return !o.exited +} + +func (o *osProcess) signalTerm() error { + return o.cmd.Process.Signal(syscall.SIGTERM) //nolint:wrapcheck +} + +func (o *osProcess) kill() error { + return o.cmd.Process.Kill() //nolint:wrapcheck +} + +func (o *osProcess) wait(timeout time.Duration) bool { + select { + case <-o.done: + return true + case <-time.After(timeout): + return false + } +} + +func (o *osProcess) pid() int { return o.cmd.Process.Pid } diff --git a/internal/serve/pool_test.go b/internal/serve/pool_test.go new file mode 100644 index 0000000..43a68c7 --- /dev/null +++ b/internal/serve/pool_test.go @@ -0,0 +1,469 @@ +package serve + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/glim-sh/cuttle/internal/fingerprint" +) + +var errFakeNoFile = errors.New("no such file") + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +type fakeProcess struct { + mu sync.Mutex + alive bool + termed bool + killed bool +} + +func newFakeProcess() *fakeProcess { return &fakeProcess{alive: true} } + +func (f *fakeProcess) running() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.alive +} + +func (f *fakeProcess) signalTerm() error { + f.mu.Lock() + defer f.mu.Unlock() + f.termed = true + f.alive = false + return nil +} + +func (f *fakeProcess) kill() error { + f.mu.Lock() + defer f.mu.Unlock() + f.killed = true + f.alive = false + return nil +} + +func (f *fakeProcess) wait(time.Duration) bool { return true } +func (f *fakeProcess) pid() int { return 4242 } + +func (f *fakeProcess) terminated() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.termed || f.killed +} + +type fakeLauncher struct { + port int + mu sync.Mutex + started [][]string + procs []*fakeProcess +} + +func (f *fakeLauncher) toLauncher() launcher { + return launcher{ + allocPort: func() (int, error) { return f.port, nil }, + start: func(_ string, args []string) (processHandle, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.started = append(f.started, args) + p := newFakeProcess() + f.procs = append(f.procs, p) + return p, nil + }, + waitReady: func(context.Context, int) bool { return true }, + } +} + +func (f *fakeLauncher) lastArgs() []string { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.started) == 0 { + return nil + } + return f.started[len(f.started)-1] +} + +func (f *fakeLauncher) launchCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.procs) +} + +func newTestPool(t *testing.T, cfg serveConfig, l launcher) *chromePool { + t.Helper() + cfg.dataDir = t.TempDir() + cfg.headless = true + return newChromePool(cfg, "/fake/chrome", nil, l, fingerprint.GeoResolver{}) +} + +// --------------------------------------------------------------------------- +// Pool behavior +// --------------------------------------------------------------------------- + +func TestGetOrLaunchDefaultProxyInheritance(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{proxy: "http://bob:secret@proxy.example:8080"}, fl.toLauncher()) + + inst, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "s1"}) + if err != nil { + t.Fatalf("getOrLaunch: %v", err) + } + if inst.proxy != "http://bob:secret@proxy.example:8080" { + t.Errorf("instance proxy=%q (should inherit server default with creds)", inst.proxy) + } + if !slices.Contains(fl.lastArgs(), "--proxy-server=http://proxy.example:8080") { + t.Errorf("chrome args missing cred-stripped proxy-server: %v", fl.lastArgs()) + } + // The credentials must NOT reach the argv (answered over CDP instead). + for _, a := range fl.lastArgs() { + if strings.Contains(a, "secret") { + t.Errorf("credentials leaked into argv: %q", a) + } + } +} + +func TestGetOrLaunchExplicitProxyOverridesDefault(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{proxy: "http://default:pw@def.example:8080"}, fl.toLauncher()) + + inst, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "s1", proxy: "socks5://per.example:1080"}) + if err != nil { + t.Fatalf("getOrLaunch: %v", err) + } + if inst.proxy != "socks5://per.example:1080" { + t.Errorf("per-connection proxy should win, got %q", inst.proxy) + } +} + +func TestGetOrLaunchInvalidSeed(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{}, fl.toLauncher()) + + _, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "bad seed!"}) + var le *launchError + if !errors.As(err, &le) || le.status != http.StatusBadRequest { + t.Fatalf("want 400 launchError, got %v", err) + } + if fl.launchCount() != 0 { + t.Errorf("invalid seed must not launch a process") + } +} + +func TestGetOrLaunchReuseFirstLaunchWins(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{}, fl.toLauncher()) + + a, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "s1"}) + if err != nil { + t.Fatal(err) + } + b, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "s1", proxy: "http://ignored:1"}) + if err != nil { + t.Fatal(err) + } + if a != b { + t.Errorf("second call should return the same running instance") + } + if fl.launchCount() != 1 { + t.Errorf("launchCount=%d want 1 (first-launch wins)", fl.launchCount()) + } + if a.proxy != "" { + t.Errorf("first launch had no proxy; later proxy must be ignored, got %q", a.proxy) + } +} + +func TestIdleReap(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{idleTimeout: 30 * time.Millisecond}, fl.toLauncher()) + + inst, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "s1"}) + if err != nil { + t.Fatal(err) + } + fp := inst.process.(*fakeProcess) + + pool.connect("s1") + pool.disconnect("s1") + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + pool.mu.Lock() + _, present := pool.processes["s1"] + pool.mu.Unlock() + if !present { + break + } + time.Sleep(5 * time.Millisecond) + } + pool.mu.Lock() + _, present := pool.processes["s1"] + pool.mu.Unlock() + if present { + t.Fatal("idle process was not reaped") + } + if !fp.terminated() { + t.Error("reaped process was not terminated (SIGTERM)") + } +} + +func TestNoReapWhenIdleTimeoutZero(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{idleTimeout: 0}, fl.toLauncher()) + + inst, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "s1"}) + if err != nil { + t.Fatal(err) + } + pool.connect("s1") + pool.disconnect("s1") + + time.Sleep(60 * time.Millisecond) + pool.mu.Lock() + _, present := pool.processes["s1"] + pool.mu.Unlock() + if !present { + t.Fatal("process must not be reaped when idle-timeout is 0") + } + if inst.process.(*fakeProcess).terminated() { + t.Error("process must stay alive with idle-timeout 0") + } +} + +func TestShutdownTerminatesAll(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{}, fl.toLauncher()) + + for _, seed := range []string{"a", "b", "c"} { + if _, err := pool.getOrLaunch(context.Background(), connectRequest{seed: seed}); err != nil { + t.Fatal(err) + } + } + pool.shutdown() + fl.mu.Lock() + procs := slices.Clone(fl.procs) + fl.mu.Unlock() + for _, p := range procs { + if !p.terminated() { + t.Error("shutdown left a process running") + } + } + pool.mu.Lock() + n := len(pool.processes) + pool.mu.Unlock() + if n != 0 { + t.Errorf("processes not cleared: %d", n) + } +} + +func TestEphemeralProfileDir(t *testing.T) { + t.Parallel() + fl := &fakeLauncher{port: 5100} + pool := newTestPool(t, serveConfig{ephemeral: true}, fl.toLauncher()) + + inst, err := pool.getOrLaunch(context.Background(), connectRequest{seed: "s1"}) + if err != nil { + t.Fatal(err) + } + // Ephemeral dir is a fresh scratch under dataDir, not the stable seed path. + if inst.userDataDir == pool.dataDir+"/s1" { + t.Errorf("ephemeral dir should be a scratch path, got %q", inst.userDataDir) + } + if !strings.HasPrefix(inst.userDataDir, pool.dataDir) { + t.Errorf("ephemeral dir %q not under dataDir %q", inst.userDataDir, pool.dataDir) + } +} + +// --------------------------------------------------------------------------- +// HTTP endpoints against a fake CDP backend +// --------------------------------------------------------------------------- + +type fakeCDP struct { + server *httptest.Server + port int +} + +func newFakeCDP(t *testing.T) *fakeCDP { + t.Helper() + f := &fakeCDP{} + mux := http.NewServeMux() + mux.HandleFunc("GET /json/version", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"Browser":"Chrome/148","webSocketDebuggerUrl":"ws://127.0.0.1:` + + strconv.Itoa(f.port) + `/devtools/browser/GUID123"}`)) + }) + mux.HandleFunc("GET /json/list", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[{"type":"page","webSocketDebuggerUrl":"ws://127.0.0.1:` + + strconv.Itoa(f.port) + `/devtools/page/PAGE9"}]`)) + }) + mux.HandleFunc("GET /devtools/{path...}", func(w http.ResponseWriter, r *http.Request) { + c, err := websocket.Accept(w, r, &websocket.AcceptOptions{InsecureSkipVerify: true}) + if err != nil { + return + } + c.SetReadLimit(-1) + ctx := r.Context() + _ = c.Write(ctx, websocket.MessageText, []byte(`{"method":"CDP.greeting"}`)) + for { + typ, data, err := c.Read(ctx) + if err != nil { + return + } + _ = c.Write(ctx, typ, append([]byte("echo:"), data...)) + } + }) + f.server = httptest.NewServer(mux) + t.Cleanup(f.server.Close) + u, err := url.Parse(f.server.URL) + if err != nil { + t.Fatal(err) + } + f.port, _ = strconv.Atoi(u.Port()) + return f +} + +func TestHandleJSONVersionHostRewrite(t *testing.T) { + t.Parallel() + cdp := newFakeCDP(t) + fl := &fakeLauncher{port: cdp.port} + pool := newTestPool(t, serveConfig{}, fl.toLauncher()) + m := &multiplexer{pool: pool, port: 9222} + + tests := []struct { + name string + target string + host string + headers map[string]string + want string + }{ + { + name: "seed rewrites to request host", + target: "/json/version?fingerprint=seedX", + host: "myhost:1234", + want: "ws://myhost:1234/fingerprint/seedX/devtools/browser/GUID123", + }, + { + name: "no seed uses default devtools path", + target: "/json/version", + host: "10.1.2.3:9222", + want: "ws://10.1.2.3:9222/devtools/browser/GUID123", + }, + { + name: "x-forwarded-host and https -> wss", + target: "/json/version?fingerprint=seedX", + host: "internal:9222", + headers: map[string]string{"X-Forwarded-Host": "public.example", "X-Forwarded-Proto": "https"}, + want: "wss://public.example/fingerprint/seedX/devtools/browser/GUID123", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, tc.target, nil) + req.Host = tc.host + for k, v := range tc.headers { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + m.handleJSONVersion(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `"webSocketDebuggerUrl":"`+tc.want+`"`) { + t.Errorf("body=%s\nwant ws url %q", rec.Body.String(), tc.want) + } + }) + } +} + +func TestHandleJSONListHostRewrite(t *testing.T) { + t.Parallel() + cdp := newFakeCDP(t) + fl := &fakeLauncher{port: cdp.port} + pool := newTestPool(t, serveConfig{}, fl.toLauncher()) + m := &multiplexer{pool: pool, port: 9222} + + req := httptest.NewRequest(http.MethodGet, "/json/list?fingerprint=seedX", nil) + req.Host = "myhost:1234" + rec := httptest.NewRecorder() + m.handleJSONList(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + want := "ws://myhost:1234/fingerprint/seedX/devtools/page/PAGE9" + if !strings.Contains(rec.Body.String(), want) { + t.Errorf("body=%s\nwant %q", rec.Body.String(), want) + } +} + +// --------------------------------------------------------------------------- +// Bidirectional WebSocket frame piping through the multiplexer +// --------------------------------------------------------------------------- + +func TestWSFramePipingBothWays(t *testing.T) { + t.Parallel() + cdp := newFakeCDP(t) + fl := &fakeLauncher{port: cdp.port} + pool := newTestPool(t, serveConfig{}, fl.toLauncher()) + m := &multiplexer{pool: pool, port: 9222} + + front := httptest.NewServer(m.routes()) + t.Cleanup(front.Close) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + wsURL := "ws" + strings.TrimPrefix(front.URL, "http") + "/fingerprint/s1/devtools/browser/GUID123" + client, _, err := websocket.Dial(ctx, wsURL, nil) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = client.Close(websocket.StatusNormalClosure, "") }() + + // cdp -> client: the fake backend pushes a greeting on connect. + typ, data, err := client.Read(ctx) + if err != nil { + t.Fatalf("read greeting: %v", err) + } + if typ != websocket.MessageText || string(data) != `{"method":"CDP.greeting"}` { + t.Fatalf("unexpected greeting: %q", data) + } + + // client -> cdp -> client: our frame reaches Chrome and its reply comes back. + if werr := client.Write(ctx, websocket.MessageText, []byte(`{"id":1,"method":"Browser.getVersion"}`)); werr != nil { + t.Fatalf("write: %v", werr) + } + _, echo, err := client.Read(ctx) + if err != nil { + t.Fatalf("read echo: %v", err) + } + if string(echo) != `echo:{"id":1,"method":"Browser.getVersion"}` { + t.Errorf("echo=%q", echo) + } + + // The live session is refcounted on the seed. + pool.mu.Lock() + conns := pool.conns["s1"] + pool.mu.Unlock() + if conns != 1 { + t.Errorf("connections=%d want 1", conns) + } +} diff --git a/internal/serve/serve.go b/internal/serve/serve.go new file mode 100644 index 0000000..20c63ba --- /dev/null +++ b/internal/serve/serve.go @@ -0,0 +1,343 @@ +// Package serve implements `cuttle serve`, the in-container CDP multiplexer: +// one stealth Chrome process per fingerprint seed, all fronted on one port, +// with per-connection fingerprint routing. It is a faithful port of the Python +// cuttleserve daemon plus a server-level default proxy and ephemeral profile +// dirs. +package serve + +import ( + "context" + "encoding/json" + "errors" + "log" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/spf13/cobra" + + "github.com/glim-sh/cuttle/internal/cli" + "github.com/glim-sh/cuttle/internal/fingerprint" +) + +func init() { cli.AddCommand(newServeCmd()) } + +var logger = log.New(os.Stderr, "", log.Ltime) + +func logInfo(format string, args ...any) { logger.Printf("INFO "+format, args...) } +func logWarn(format string, args ...any) { logger.Printf("WARN "+format, args...) } +func logError(format string, args ...any) { logger.Printf("ERROR "+format, args...) } + +const ( + defaultPort = 9222 + basePort = 5100 + terminateGrace = 5 * time.Second + shutdownGrace = 10 * time.Second + reservedSeed = fingerprint.ReservedSeed + proxyEnv = "CUTTLESERVE_PROXY" + ephemeralEnv = "CUTTLESERVE_EPHEMERAL" + idleTimeoutEnv = "CLOAKSERVE_IDLE_TIMEOUT" + hostEnv = "CUTTLESERVE_HOST" + readHeaderLimit = 10 * time.Second +) + +var ( + errIdleTimeoutNegative = errors.New("--idle-timeout must be greater than or equal to 0") + errInvalidDefaultSeed = errors.New("invalid --fingerprint seed") +) + +// baseChromeArgs run Chrome directly (outside Playwright); Playwright normally +// adds its own version of these. +var baseChromeArgs = []string{ + "--no-first-run", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-extensions", + "--disable-popup-blocking", + "--disable-background-networking", + "--metrics-recording-only", + "--ignore-gpu-blocklist", +} + +func validSeed(seed string) bool { + return fingerprint.ValidSeed(seed) +} + +// serveConfig holds the parsed cuttleserve flags. +type serveConfig struct { + port int + headless bool + dataDir string + defaultSeed string + defaultLocale string + defaultTimezone string + idleTimeout time.Duration + keepProfile bool + proxy string + ephemeral bool +} + +func newServeCmd() *cobra.Command { + return &cobra.Command{ + Use: "serve [flags] [-- chrome-flags...]", + Short: "Run the in-container CDP multiplexer (image entrypoint)", + DisableFlagParsing: true, + RunE: func(cmd *cobra.Command, args []string) error { + for _, a := range args { + if a == "-h" || a == "--help" { + return cmd.Help() + } + } + return run(cmd.Context(), args) + }, + } +} + +func run(ctx context.Context, argv []string) error { + cfg, globalArgs, err := parseCLIArgs(argv) + if err != nil { + return err + } + + binary, err := fingerprint.EnsureBinary() + if err != nil { + return err + } + + if cfg.defaultSeed != "" && !validSeed(cfg.defaultSeed) { + return errInvalidDefaultSeed + } + + pool := newChromePool(cfg, binary, globalArgs, defaultLauncher(), fingerprint.NewGeoResolver()) + mux := (&multiplexer{pool: pool, port: cfg.port}).routes() + + host := bindHost(defaultEnvProbe()) + httpServer := &http.Server{ + Addr: net.JoinHostPort(host, strconv.Itoa(cfg.port)), + Handler: mux, + ReadHeaderTimeout: readHeaderLimit, + } + + ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + logInfo("CDP multiplexer starting on %s:%d", host, cfg.port) + serveErr := make(chan error, 1) + go func() { + if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + serveErr <- err + return + } + serveErr <- nil + }() + + select { + case err := <-serveErr: + pool.shutdown() + return err + case <-ctx.Done(): + } + + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGrace) + defer cancel() + _ = httpServer.Shutdown(shutdownCtx) + pool.shutdown() + return nil +} + +// parseCLIArgs mirrors the Python cuttleserve arg parser: it extracts the +// daemon's own flags and returns the remaining args as Chrome passthrough. +// --fingerprint / --fingerprint-locale / --fingerprint-timezone become config +// defaults so they route through build_args (locale needs both --lang and +// --fingerprint-locale). Query-string params override these per-connection. +func parseCLIArgs(argv []string) (serveConfig, []string, error) { + cfg := serveConfig{ + port: defaultPort, + headless: true, + idleTimeout: defaultIdleTimeout(), + proxy: os.Getenv(proxyEnv), + ephemeral: parseBoolEnv(os.Getenv(ephemeralEnv)), + } + passthrough := []string{} + consumedPrefixes := []string{ + "--port=", + "--data-dir=", + "--idle-timeout=", + "--remote-debugging-port=", + "--remote-debugging-address=", + } + + for _, arg := range argv { + switch { + case strings.HasPrefix(arg, "--port="): + p, err := strconv.Atoi(strings.SplitN(arg, "=", 2)[1]) + if err != nil { + return serveConfig{}, nil, errors.New("invalid --port value") //nolint:err113 + } + cfg.port = p + case strings.HasPrefix(arg, "--data-dir="): + cfg.dataDir = strings.SplitN(arg, "=", 2)[1] + case strings.HasPrefix(arg, "--idle-timeout="): + d, err := parseIdleTimeout(strings.SplitN(arg, "=", 2)[1]) + if err != nil { + return serveConfig{}, nil, err + } + cfg.idleTimeout = d + case strings.HasPrefix(arg, "--proxy="): + cfg.proxy = strings.SplitN(arg, "=", 2)[1] + case arg == "--ephemeral": + cfg.ephemeral = true + case arg == "--headless=false" || arg == "--headless=False": + cfg.headless = false + passthrough = append(passthrough, arg) + case arg == "--keep-profile": + cfg.keepProfile = true + case hasAnyPrefix(arg, consumedPrefixes): + // Strip silently. + case strings.HasPrefix(arg, "--fingerprint-locale="): + cfg.defaultLocale = strings.SplitN(arg, "=", 2)[1] + case strings.HasPrefix(arg, "--fingerprint-timezone="): + cfg.defaultTimezone = strings.SplitN(arg, "=", 2)[1] + case strings.HasPrefix(arg, "--fingerprint="): + cfg.defaultSeed = strings.SplitN(arg, "=", 2)[1] + default: + passthrough = append(passthrough, arg) + } + } + + if cfg.dataDir == "" { + cfg.dataDir = defaultDataDir(defaultEnvProbe()) + } + return cfg, passthrough, nil +} + +func hasAnyPrefix(s string, prefixes []string) bool { + for _, p := range prefixes { + if strings.HasPrefix(s, p) { + return true + } + } + return false +} + +func parseBoolEnv(v string) bool { + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +func defaultIdleTimeout() time.Duration { + v, ok := os.LookupEnv(idleTimeoutEnv) + if !ok { + return 0 + } + d, err := parseIdleTimeout(v) + if err != nil { + return 0 + } + return d +} + +func parseIdleTimeout(value string) (time.Duration, error) { + value = strings.TrimSpace(value) + switch strings.ToLower(value) { + case "0", "false", "off", "none", "disabled": + return 0, nil + } + seconds, err := strconv.ParseFloat(value, 64) + if err != nil { + return 0, errors.New("invalid --idle-timeout value") //nolint:err113 + } + if seconds < 0 { + return 0, errIdleTimeoutNegative + } + return time.Duration(seconds * float64(time.Second)), nil +} + +// envProbe abstracts the filesystem/env reads that drive container detection so +// they can be faked in tests. +type envProbe struct { + stat func(string) bool + getenv func(string) string + readFile func(string) ([]byte, error) + homeDir func() (string, error) +} + +func defaultEnvProbe() envProbe { + return envProbe{ + stat: func(path string) bool { + _, err := os.Stat(path) + return err == nil + }, + getenv: os.Getenv, + readFile: os.ReadFile, + homeDir: os.UserHomeDir, + } +} + +// inContainer reports whether the process runs inside a container (docker, +// podman, or k8s/containerd). The plain-file markers /.dockerenv and +// /run/.containerenv are docker/podman-only and BOTH are absent under +// Kubernetes+containerd, which would silently pin the CDP listener to loopback +// and refuse every cross-pod client; the KUBERNETES_SERVICE_HOST env and the +// container cgroup close that gap. +func (e envProbe) inContainer() bool { + if e.stat("/.dockerenv") || e.stat("/run/.containerenv") { + return true + } + if e.getenv("KUBERNETES_SERVICE_HOST") != "" { + return true + } + data, err := e.readFile("/proc/1/cgroup") + if err != nil { + return false + } + cgroup := string(data) + for _, marker := range []string{"kubepods", "docker", "containerd", "crio"} { + if strings.Contains(cgroup, marker) { + return true + } + } + return false +} + +// bindHost binds 0.0.0.0 in a container so cross-pod/host clients can reach the +// multiplexer, and loopback-only on bare metal. CUTTLESERVE_HOST overrides. +func bindHost(e envProbe) string { + if h := e.getenv(hostEnv); h != "" { + return h + } + if e.inContainer() { + return "0.0.0.0" + } + return "127.0.0.1" +} + +func defaultDataDir(e envProbe) string { + if e.inContainer() { + return "/tmp/cuttle" + } + if dir := e.getenv("XDG_DATA_HOME"); dir != "" { + return filepath.Join(dir, "cuttle", "serve") + } + home, err := e.homeDir() + if err != nil { + return "/tmp/cuttle" + } + return filepath.Join(home, ".local", "share", "cuttle", "serve") +} + +func writeJSON(w http.ResponseWriter, status int, payload any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} diff --git a/internal/serve/serve_test.go b/internal/serve/serve_test.go new file mode 100644 index 0000000..6b08d56 --- /dev/null +++ b/internal/serve/serve_test.go @@ -0,0 +1,243 @@ +package serve + +import ( + "testing" + "time" +) + +func TestParseIdleTimeout(t *testing.T) { + t.Parallel() + tests := []struct { + name string + in string + want time.Duration + wantErr bool + }{ + {"zero", "0", 0, false}, + {"disabled-word", "disabled", 0, false}, + {"off", "off", 0, false}, + {"none", "none", 0, false}, + {"seconds", "30", 30 * time.Second, false}, + {"fractional", "1.5", 1500 * time.Millisecond, false}, + {"negative", "-1", 0, true}, + {"garbage", "soon", 0, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := parseIdleTimeout(tc.in) + if (err != nil) != tc.wantErr { + t.Fatalf("err=%v wantErr=%v", err, tc.wantErr) + } + if err == nil && got != tc.want { + t.Fatalf("got %v want %v", got, tc.want) + } + }) + } +} + +func TestParseCLIArgs(t *testing.T) { + t.Setenv(proxyEnv, "") + t.Setenv(ephemeralEnv, "") + t.Setenv(idleTimeoutEnv, "") + t.Setenv("HOME", "/home/tester") + + cfg, passthrough, err := parseCLIArgs([]string{ + "--port=9333", + "--data-dir=/data", + "--idle-timeout=45", + "--keep-profile", + "--ephemeral", + "--proxy=http://user:pass@proxy.example:8080", + "--fingerprint=abc", + "--fingerprint-locale=en-GB", + "--fingerprint-timezone=Europe/London", + "--headless=false", + "--remote-debugging-port=1", // consumed, stripped + "--some-chrome-flag", + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if cfg.port != 9333 { + t.Errorf("port=%d want 9333", cfg.port) + } + if cfg.dataDir != "/data" { + t.Errorf("dataDir=%q", cfg.dataDir) + } + if cfg.idleTimeout != 45*time.Second { + t.Errorf("idleTimeout=%v", cfg.idleTimeout) + } + if !cfg.keepProfile || !cfg.ephemeral { + t.Errorf("keepProfile=%v ephemeral=%v", cfg.keepProfile, cfg.ephemeral) + } + if cfg.proxy != "http://user:pass@proxy.example:8080" { + t.Errorf("proxy=%q", cfg.proxy) + } + if cfg.defaultSeed != "abc" || cfg.defaultLocale != "en-GB" || cfg.defaultTimezone != "Europe/London" { + t.Errorf("fingerprint defaults: %q %q %q", cfg.defaultSeed, cfg.defaultLocale, cfg.defaultTimezone) + } + if cfg.headless { + t.Errorf("headless should be false") + } + // --headless=false and the unknown chrome flag pass through; consumed flags do not. + wantPass := []string{"--headless=false", "--some-chrome-flag"} + if len(passthrough) != len(wantPass) { + t.Fatalf("passthrough=%v want %v", passthrough, wantPass) + } + for i := range wantPass { + if passthrough[i] != wantPass[i] { + t.Fatalf("passthrough=%v want %v", passthrough, wantPass) + } + } +} + +func TestParseCLIArgsEnvDefaults(t *testing.T) { + t.Setenv(proxyEnv, "http://env-proxy:3128") + t.Setenv(ephemeralEnv, "true") + t.Setenv(idleTimeoutEnv, "60") + t.Setenv("HOME", "/home/tester") + + cfg, _, err := parseCLIArgs(nil) + if err != nil { + t.Fatalf("err: %v", err) + } + if cfg.proxy != "http://env-proxy:3128" { + t.Errorf("proxy from env=%q", cfg.proxy) + } + if !cfg.ephemeral { + t.Errorf("ephemeral from env not set") + } + if cfg.idleTimeout != 60*time.Second { + t.Errorf("idleTimeout from env=%v", cfg.idleTimeout) + } + // A CLI flag overrides the env default. + cfg2, _, _ := parseCLIArgs([]string{"--proxy=http://cli-proxy:8888"}) + if cfg2.proxy != "http://cli-proxy:8888" { + t.Errorf("cli proxy override=%q", cfg2.proxy) + } +} + +func TestValidSeed(t *testing.T) { + t.Parallel() + tests := []struct { + seed string + want bool + }{ + {"abc123", true}, + {"seed_with-dashes", true}, + {"__default__", false}, + {"", false}, + {"has space", false}, + {"has/slash", false}, + {"has.dot", false}, + } + for _, tc := range tests { + if got := validSeed(tc.seed); got != tc.want { + t.Errorf("validSeed(%q)=%v want %v", tc.seed, got, tc.want) + } + } +} + +func TestBindHost(t *testing.T) { + t.Parallel() + tests := []struct { + name string + probe envProbe + want string + }{ + { + name: "env override wins", + probe: envProbe{ + getenv: func(k string) string { + if k == hostEnv { + return "10.0.0.5" + } + return "" + }, + stat: func(string) bool { return true }, + readFile: func(string) ([]byte, error) { return nil, nil }, + }, + want: "10.0.0.5", + }, + { + name: "dockerenv marker -> 0.0.0.0", + probe: envProbe{ + getenv: func(string) string { return "" }, + stat: func(p string) bool { return p == "/.dockerenv" }, + readFile: func(string) ([]byte, error) { return nil, errFakeNoFile }, + }, + want: "0.0.0.0", + }, + { + name: "kubernetes env -> 0.0.0.0", + probe: envProbe{ + getenv: func(k string) string { + if k == "KUBERNETES_SERVICE_HOST" { + return "10.96.0.1" + } + return "" + }, + stat: func(string) bool { return false }, + readFile: func(string) ([]byte, error) { return nil, errFakeNoFile }, + }, + want: "0.0.0.0", + }, + { + name: "containerd cgroup (no marker files) -> 0.0.0.0", + probe: envProbe{ + getenv: func(string) string { return "" }, + stat: func(string) bool { return false }, + readFile: func(string) ([]byte, error) { return []byte("0::/kubepods/pod123/abc"), nil }, + }, + want: "0.0.0.0", + }, + { + name: "bare metal -> loopback", + probe: envProbe{ + getenv: func(string) string { return "" }, + stat: func(string) bool { return false }, + readFile: func(string) ([]byte, error) { return []byte("0::/user.slice/session.scope"), nil }, + }, + want: "127.0.0.1", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := bindHost(tc.probe); got != tc.want { + t.Errorf("bindHost=%q want %q", got, tc.want) + } + }) + } +} + +func TestDefaultDataDir(t *testing.T) { + t.Parallel() + container := envProbe{ + getenv: func(string) string { return "" }, + stat: func(p string) bool { return p == "/.dockerenv" }, + readFile: func(string) ([]byte, error) { return nil, errFakeNoFile }, + } + if got := defaultDataDir(container); got != "/tmp/cuttle" { + t.Errorf("container dataDir=%q", got) + } + bare := envProbe{ + getenv: func(string) string { return "" }, + stat: func(string) bool { return false }, + readFile: func(string) ([]byte, error) { return nil, errFakeNoFile }, + homeDir: func() (string, error) { return "/home/tester", nil }, + } + if got := defaultDataDir(bare); got != "/home/tester/.local/share/cuttle/serve" { + t.Errorf("bare dataDir=%q", got) + } + xdgSet := envProbe{ + getenv: func(k string) string { return map[string]string{"XDG_DATA_HOME": "/xdg/data"}[k] }, + stat: func(string) bool { return false }, + readFile: func(string) ([]byte, error) { return nil, errFakeNoFile }, + homeDir: func() (string, error) { return "/home/tester", nil }, + } + if got := defaultDataDir(xdgSet); got != "/xdg/data/cuttle/serve" { + t.Errorf("xdg dataDir=%q", got) + } +} diff --git a/internal/serve/wsproxy.go b/internal/serve/wsproxy.go new file mode 100644 index 0000000..d5b5b9d --- /dev/null +++ b/internal/serve/wsproxy.go @@ -0,0 +1,308 @@ +package serve + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strconv" + "sync" + "time" + + "github.com/coder/websocket" + + "github.com/glim-sh/cuttle/internal/fingerprint" +) + +// injectedIDBase is the CDP command-id floor for our transparent proxy-auth +// commands - far above any id a real client uses, so their responses are +// recognizable and swallowed rather than forwarded. +const injectedIDBase = 2_000_000_000 + +// synthBrowserContextID is stamped onto default-context service_worker targets +// (see stampSWContext). Any truthy value works: playwright looks it up, misses, +// and falls back to its default context; it never resolves to a real id. +const synthBrowserContextID = "0000000000000000000000000000CA5E" + +const wsReadLimit = -1 // disable coder/websocket's default message size cap + +func (m *multiplexer) handleWSSeed(w http.ResponseWriter, r *http.Request) { + if m.rejectUntrustedOrigin(w, r) { + return + } + seed := r.PathValue("seed") + path := r.PathValue("path") + + cp, err := m.pool.getOrLaunch(r.Context(), connectRequest{seed: seed}) + if err != nil { + writeLaunchError(w, err) + return + } + _, user, pass := fingerprint.SplitProxyAuth(cp.proxy) + m.serveWS(w, r, cp, seed, "CDP seed="+seed+" ["+path+"]", path, user, pass) +} + +func (m *multiplexer) handleWSDefault(w http.ResponseWriter, r *http.Request) { + if m.rejectUntrustedOrigin(w, r) { + return + } + path := r.PathValue("path") + + cp, err := m.pool.getOrLaunch(r.Context(), connectRequest{}) + if err != nil { + writeLaunchError(w, err) + return + } + _, user, pass := fingerprint.SplitProxyAuth(cp.proxy) + m.serveWS(w, r, cp, reservedSeed, "CDP default ["+path+"]", path, user, pass) +} + +func (m *multiplexer) serveWS(w http.ResponseWriter, r *http.Request, cp *chromeInstance, seedKey, label, path, user, pass string) { + // Origin already enforced by rejectUntrustedOrigin. + clientWS, err := websocket.Accept(w, r, &websocket.AcceptOptions{InsecureSkipVerify: true}) + if err != nil { + logError("%s: accept failed: %v", label, err) + return + } + + m.pool.connect(seedKey) + defer m.pool.disconnect(seedKey) + + target := "ws://127.0.0.1:" + strconv.Itoa(cp.cdpPort) + "/devtools/" + path + proxyCDPWebsocket(r.Context(), clientWS, target, label, user, pass) +} + +// proxyCDPWebsocket pipes CDP frames between the client and the seed's Chrome. +// +// When the seed runs behind a credential-stripped --proxy-server (the forks +// reject inline creds), it transparently answers proxy 407s: the client's own +// Fetch.enable is rewritten to also handleAuthRequests, and the resulting +// Fetch.authRequired events are intercepted here and answered with the stored +// credentials over CDP - never surfaced to the client. This rides the client's +// OWN Fetch session, so it works for HTTPS CONNECT and does not conflict with +// the client's own request interception. +func proxyCDPWebsocket(ctx context.Context, clientWS *websocket.Conn, target, label, user, pass string) { + inject := user != "" + + dialCtx, dialCancel := context.WithTimeout(ctx, 10*time.Second) + cdpWS, dialResp, err := websocket.Dial(dialCtx, target, nil) + dialCancel() + if dialResp != nil && dialResp.Body != nil { + _ = dialResp.Body.Close() + } + if err != nil { + logError("%s error: %v", label, err) + _ = clientWS.Close(websocket.StatusInternalError, "cdp dial failed") + return + } + logInfo("%s: connected to %s", label, target) + clientWS.SetReadLimit(wsReadLimit) + cdpWS.SetReadLimit(wsReadLimit) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var cdpMu sync.Mutex + cdpSend := func(typ websocket.MessageType, data []byte) error { + cdpMu.Lock() + defer cdpMu.Unlock() + return cdpWS.Write(ctx, typ, data) + } + + var wg sync.WaitGroup + wg.Go(func() { + defer cancel() + for { + typ, data, err := clientWS.Read(ctx) + if err != nil { + return + } + if typ == websocket.MessageText && inject { + data = rewriteFetchEnable(data) + } + if err := cdpSend(typ, data); err != nil { + return + } + } + }) + + injectedIDs := map[int64]struct{}{} + nextInjected := int64(injectedIDBase) + for { + typ, data, err := cdpWS.Read(ctx) + if err != nil { + break + } + // Prefilter before the full JSON decode: handleProxyAuth only acts on a + // response to one of our injected commands (which exist only while + // injectedIDs is non-empty) or a Fetch.authRequired event. In steady + // state a CDP session streams thousands of other frames; skip decoding + // them, matching the bytes.Contains guard the sibling patches use. + if inject && typ == websocket.MessageText && + (len(injectedIDs) > 0 || bytes.Contains(data, []byte(`"Fetch.authRequired"`))) { + handled, cmd := handleProxyAuth(data, injectedIDs, nextInjected, user, pass) + if cmd != nil { + nextInjected++ + _ = cdpSend(websocket.MessageText, cmd) + } + if handled { + continue + } + } + if typ == websocket.MessageText { + data = stampSWContext(data) + } + if err := clientWS.Write(ctx, typ, data); err != nil { + break + } + } + + cancel() + _ = cdpWS.Close(websocket.StatusNormalClosure, "") + _ = clientWS.Close(websocket.StatusNormalClosure, "") + wg.Wait() + logInfo("%s: disconnected", label) +} + +// stampSWContext works around Chrome 148 reporting a site's service_worker +// target under the default browser context with an EMPTY browserContextId. +// playwright-core's connectOverCDP asserts that field is truthy in its +// Target.attachedToTarget handler, and the uncaught throw kills the client +// process (repro: any page that registers a service worker). Stamping a +// synthetic id makes the assert pass; playwright then falls back to its default +// context and handles the SW normally. The browser and page stay fully +// authentic - nothing in navigator is patched. Only service_worker +// attachedToTarget frames with a missing id are touched. +func stampSWContext(data []byte) []byte { + if !bytes.Contains(data, []byte(`"Target.attachedToTarget"`)) || + !bytes.Contains(data, []byte(`"service_worker"`)) { + return data + } + msg, ok := decodeCDP(data) + if !ok { + return data + } + if asString(msg["method"]) != "Target.attachedToTarget" { + return data + } + params, _ := msg["params"].(map[string]any) + targetInfo, _ := params["targetInfo"].(map[string]any) + if targetInfo == nil || asString(targetInfo["type"]) != "service_worker" { + return data + } + if bcid, ok := targetInfo["browserContextId"]; ok && asString(bcid) != "" { + return data + } + targetInfo["browserContextId"] = synthBrowserContextID + out, err := json.Marshal(msg) + if err != nil { + return data + } + return out +} + +// rewriteFetchEnable adds handleAuthRequests to a client's Fetch.enable so +// Chrome surfaces proxy 407s as Fetch.authRequired on the client's own session. +func rewriteFetchEnable(data []byte) []byte { + if !bytes.Contains(data, []byte(`"Fetch.enable"`)) { + return data + } + msg, ok := decodeCDP(data) + if !ok { + return data + } + if asString(msg["method"]) != "Fetch.enable" { + return data + } + params, _ := msg["params"].(map[string]any) + if params == nil { + params = map[string]any{} + msg["params"] = params + } + if v, ok := params["handleAuthRequests"].(bool); ok && v { + return data + } + params["handleAuthRequests"] = true + out, err := json.Marshal(msg) + if err != nil { + return data + } + return out +} + +// handleProxyAuth inspects a Chrome->client frame. It returns (swallow, command): +// a response to one of our injected commands swallows it; a Fetch.authRequired +// yields a continueWithAuth command to send and is swallowed (the client never +// asked for auth handling); anything else is forwarded untouched. +func handleProxyAuth(data []byte, injectedIDs map[int64]struct{}, cmdID int64, user, pass string) (bool, []byte) { + msg, ok := decodeCDP(data) + if !ok { + return false, nil + } + if mid, ok := asInt(msg["id"]); ok { + if _, ours := injectedIDs[mid]; ours { + delete(injectedIDs, mid) + if _, hasErr := msg["error"]; hasErr { + logWarn("proxy-auth: continueWithAuth failed: %v", msg["error"]) + } + return true, nil + } + } + if asString(msg["method"]) != "Fetch.authRequired" { + return false, nil + } + params, _ := msg["params"].(map[string]any) + challenge, _ := params["authChallenge"].(map[string]any) + var response map[string]any + if asString(challenge["source"]) == "Proxy" { + response = map[string]any{"response": "ProvideCredentials", "username": user, "password": pass} + } else { + response = map[string]any{"response": "Default"} + } + cmd := map[string]any{ + "id": cmdID, + "method": "Fetch.continueWithAuth", + "params": map[string]any{ + "requestId": params["requestId"], + "authChallengeResponse": response, + }, + } + if sid := asString(msg["sessionId"]); sid != "" { + cmd["sessionId"] = sid + } + injectedIDs[cmdID] = struct{}{} + out, err := json.Marshal(cmd) + if err != nil { + return true, nil + } + return true, out +} + +// decodeCDP unmarshals a CDP frame with number fidelity preserved (json.Number) +// so large command ids survive a re-marshal. +func decodeCDP(data []byte) (map[string]any, bool) { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + var msg map[string]any + if err := dec.Decode(&msg); err != nil { + return nil, false + } + return msg, true +} + +func asString(v any) string { + s, _ := v.(string) + return s +} + +func asInt(v any) (int64, bool) { + n, ok := v.(json.Number) + if !ok { + return 0, false + } + i, err := n.Int64() + if err != nil { + return 0, false + } + return i, true +} diff --git a/internal/serve/wsproxy_test.go b/internal/serve/wsproxy_test.go new file mode 100644 index 0000000..0683d61 --- /dev/null +++ b/internal/serve/wsproxy_test.go @@ -0,0 +1,155 @@ +package serve + +import ( + "encoding/json" + "testing" +) + +func decode(t *testing.T, b []byte) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return m +} + +func TestStampSWContext(t *testing.T) { + t.Parallel() + + t.Run("stamps empty service_worker context", func(t *testing.T) { + t.Parallel() + in := []byte(`{"method":"Target.attachedToTarget","params":{"targetInfo":{"type":"service_worker","browserContextId":""}}}`) + out := decode(t, stampSWContext(in)) + ti := out["params"].(map[string]any)["targetInfo"].(map[string]any) + if ti["browserContextId"] != synthBrowserContextID { + t.Errorf("browserContextId=%v", ti["browserContextId"]) + } + }) + + t.Run("stamps missing service_worker context", func(t *testing.T) { + t.Parallel() + in := []byte(`{"method":"Target.attachedToTarget","params":{"targetInfo":{"type":"service_worker"}}}`) + out := decode(t, stampSWContext(in)) + ti := out["params"].(map[string]any)["targetInfo"].(map[string]any) + if ti["browserContextId"] != synthBrowserContextID { + t.Errorf("browserContextId=%v", ti["browserContextId"]) + } + }) + + t.Run("leaves populated context untouched", func(t *testing.T) { + t.Parallel() + in := []byte(`{"method":"Target.attachedToTarget","params":{"targetInfo":{"type":"service_worker","browserContextId":"REAL"}}}`) + if string(stampSWContext(in)) != string(in) { + t.Errorf("should be unchanged") + } + }) + + t.Run("leaves non-service-worker untouched", func(t *testing.T) { + t.Parallel() + in := []byte(`{"method":"Target.attachedToTarget","params":{"targetInfo":{"type":"page","browserContextId":""}}}`) + if string(stampSWContext(in)) != string(in) { + t.Errorf("should be unchanged") + } + }) + + t.Run("leaves unrelated frames byte-identical", func(t *testing.T) { + t.Parallel() + in := []byte(`{"id":1,"result":{}}`) + if string(stampSWContext(in)) != string(in) { + t.Errorf("should be unchanged") + } + }) +} + +func TestRewriteFetchEnable(t *testing.T) { + t.Parallel() + + t.Run("adds handleAuthRequests", func(t *testing.T) { + t.Parallel() + out := decode(t, rewriteFetchEnable([]byte(`{"id":5,"method":"Fetch.enable","params":{}}`))) + if out["params"].(map[string]any)["handleAuthRequests"] != true { + t.Errorf("handleAuthRequests not set: %v", out) + } + }) + + t.Run("adds params when absent", func(t *testing.T) { + t.Parallel() + out := decode(t, rewriteFetchEnable([]byte(`{"id":5,"method":"Fetch.enable"}`))) + if out["params"].(map[string]any)["handleAuthRequests"] != true { + t.Errorf("handleAuthRequests not set: %v", out) + } + }) + + t.Run("already-true left byte-identical", func(t *testing.T) { + t.Parallel() + in := []byte(`{"id":5,"method":"Fetch.enable","params":{"handleAuthRequests":true}}`) + if string(rewriteFetchEnable(in)) != string(in) { + t.Errorf("should be unchanged") + } + }) + + t.Run("non-fetch untouched", func(t *testing.T) { + t.Parallel() + in := []byte(`{"id":5,"method":"Page.enable"}`) + if string(rewriteFetchEnable(in)) != string(in) { + t.Errorf("should be unchanged") + } + }) +} + +func TestHandleProxyAuth(t *testing.T) { + t.Parallel() + + t.Run("proxy challenge answered with credentials and swallowed", func(t *testing.T) { + t.Parallel() + in := []byte(`{"method":"Fetch.authRequired","sessionId":"S1","params":{"requestId":"R1","authChallenge":{"source":"Proxy"}}}`) + swallow, cmd := handleProxyAuth(in, map[int64]struct{}{}, injectedIDBase, "bob", "secret") + if !swallow { + t.Fatal("authRequired must be swallowed") + } + out := decode(t, cmd) + if out["method"] != "Fetch.continueWithAuth" || out["sessionId"] != "S1" { + t.Errorf("cmd=%v", out) + } + resp := out["params"].(map[string]any)["authChallengeResponse"].(map[string]any) + if resp["response"] != "ProvideCredentials" || resp["username"] != "bob" || resp["password"] != "secret" { + t.Errorf("auth response=%v", resp) + } + }) + + t.Run("non-proxy challenge answered with default", func(t *testing.T) { + t.Parallel() + in := []byte(`{"method":"Fetch.authRequired","params":{"requestId":"R1","authChallenge":{"source":"Server"}}}`) + swallow, cmd := handleProxyAuth(in, map[int64]struct{}{}, injectedIDBase, "bob", "secret") + if !swallow { + t.Fatal("must swallow") + } + resp := decode(t, cmd)["params"].(map[string]any)["authChallengeResponse"].(map[string]any) + if resp["response"] != "Default" { + t.Errorf("want Default response, got %v", resp) + } + }) + + t.Run("our injected response swallowed, no command", func(t *testing.T) { + t.Parallel() + ids := map[int64]struct{}{injectedIDBase: {}} + in := []byte(`{"id":2000000000,"result":{}}`) + swallow, cmd := handleProxyAuth(in, ids, injectedIDBase+1, "bob", "secret") + if !swallow || cmd != nil { + t.Errorf("swallow=%v cmd=%v", swallow, cmd) + } + if _, ok := ids[injectedIDBase]; ok { + t.Errorf("injected id should be discarded") + } + }) + + t.Run("ordinary frame forwarded", func(t *testing.T) { + t.Parallel() + in := []byte(`{"id":7,"result":{"ok":true}}`) + swallow, cmd := handleProxyAuth(in, map[int64]struct{}{}, injectedIDBase, "bob", "secret") + if swallow || cmd != nil { + t.Errorf("ordinary frame must pass through: swallow=%v", swallow) + } + }) +} diff --git a/internal/xdg/xdg.go b/internal/xdg/xdg.go new file mode 100644 index 0000000..265c0b2 --- /dev/null +++ b/internal/xdg/xdg.go @@ -0,0 +1,26 @@ +// Package xdg resolves XDG base directories with home-directory fallbacks, so +// cuttle's config and data paths are derived in one place instead of each +// package repeating the fallback ladder. +package xdg + +import ( + "os" + "path/filepath" +) + +// ConfigDir is $XDG_CONFIG_HOME, or ~/.config when unset. It returns "" only if +// the env var is unset and the home directory cannot be determined. +func ConfigDir() string { return baseDir("XDG_CONFIG_HOME", ".config") } + +// DataDir is $XDG_DATA_HOME, or ~/.local/share when unset. +func DataDir() string { return baseDir("XDG_DATA_HOME", filepath.Join(".local", "share")) } + +func baseDir(env, fallback string) string { + if dir := os.Getenv(env); dir != "" { + return dir + } + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, fallback) + } + return "" +} diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..194fd1d --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,19 @@ +pre-commit: + piped: true + commands: + fmt: + glob: "*.go" + run: gofumpt -w {staged_files} + stage_fixed: true + lint: + glob: "*.go" + run: golangci-lint run --fix {staged_files} + stage_fixed: true + mod-tidy: + glob: "*.{go,mod,sum}" + run: go mod tidy + +pre-push: + commands: + test: + run: go test -race ./... diff --git a/ops/config/goreleaser.yaml b/ops/config/goreleaser.yaml new file mode 100644 index 0000000..938158c --- /dev/null +++ b/ops/config/goreleaser.yaml @@ -0,0 +1,66 @@ +# GoReleaser builds the cross-platform `cuttle` CLI and publishes it to the +# GitHub release (created by release-please) plus a Homebrew cask. The container +# image is built separately in the release job of ci.yml (heavy multi-stage +# Dockerfile). +version: 2 + +project_name: cuttle + +before: + hooks: + - go mod tidy + +builds: + - id: cuttle + main: ./cmd/cuttle + binary: cuttle + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w -X github.com/glim-sh/cuttle/internal/cli.version={{ .Version }} + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + +archives: + - id: cuttle + formats: + - tar.gz + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + +checksum: + name_template: checksums.txt + +# release-please already created the GitHub release; append the built artifacts. +release: + mode: append + +# release-please owns CHANGELOG.md and the release notes. +changelog: + disable: true + +# A pre-compiled binary is a cask in modern Homebrew (the `brews` formula section +# is deprecated). GH_RELEASE_TOKEN can push to the separate tap repo. +homebrew_casks: + - name: cuttle + binaries: + - cuttle + homepage: https://github.com/glim-sh/cuttle + description: Host CLI for the cuttle stealth-Chromium CDP farm + repository: + owner: tenequm + name: homebrew-tap + token: "{{ .Env.GH_RELEASE_TOKEN }}" + # The binary is unsigned; strip the quarantine bit so it runs without the + # "damaged" Gatekeeper prompt. + hooks: + post: + install: | + if OS.mac? + system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/cuttle"] + end diff --git a/Dockerfile b/ops/docker/Dockerfile similarity index 60% rename from Dockerfile rename to ops/docker/Dockerfile index 26549c0..e018b58 100644 --- a/Dockerfile +++ b/ops/docker/Dockerfile @@ -1,23 +1,39 @@ # cuttle: a stealth-Chromium CDP farm. # -# A patched CDP multiplexer (bin/cuttleserve) spawns one stealth Chrome per -# fingerprint seed, routing per-seed identity (fingerprint, proxy, geoip) over -# CDP. The Chrome engine is a FREE, redistributable stealth-Chromium fork -# (clark, MIT, default; clearcote, BSD-3, fallback) baked in as a prebuilt -# binary and selected via CLOAKBROWSER_BINARY_PATH. No proprietary binary. +# The CDP multiplexer (`cuttle serve`) spawns one stealth Chrome per fingerprint +# seed, routing per-seed identity (fingerprint, proxy, geoip) over CDP. The +# Chrome engine is a FREE, redistributable stealth-Chromium fork (clark, MIT, +# default; clearcote, BSD-3, fallback) baked in as a prebuilt binary and selected +# via CLOAKBROWSER_BINARY_PATH. No proprietary binary. # -# Clean base: FROM python:3.12-slim with zero CloakBrowser binary in any layer. -# linux/amd64 only: clark/clearcote ship linux-x64 prebuilts. On an Apple Silicon -# host the image runs emulated (fine for local dev + login handoff); production -# runs it native on an amd64 server. The Python multiplexer itself is arch-agnostic. -FROM python:3.12-slim +# The daemon is a single static Go binary and the whole build is Python-free. +# linux/amd64 only: clark/clearcote ship linux-x64 prebuilts. On +# an Apple Silicon host the image runs emulated (fine for local dev + login +# handoff); production runs it native on an amd64 server. + +# --- Go builder: the static `cuttle` binary. ------------------------------- +FROM golang:1.26-trixie AS builder +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -trimpath \ + -ldflags="-s -w -X github.com/glim-sh/cuttle/internal/cli.version=${VERSION}" \ + -o /out/cuttle ./cmd/cuttle + +# --- Runtime: Python-free stealth-Chromium farm. --------------------------- +# Base is Debian trixie (same OS as golang:1.26-trixie and python:3.12-slim), so +# the KasmVNC bookworm .deb and the font families match across stages. +FROM debian:trixie-slim # Chromium system libs + headed-mode stack (Xvfb/openbox: headed Chrome is # REQUIRED to clear escalated anti-bot challenges) + fontconfig + base fonts + # metric-compatible font families for the Windows font pack + xz (clearcote # ships .tar.xz) + X debug tools (xwininfo/xdpyinfo/scrot: verify window -# mapping and grab the :99 display in-container). No nodejs (cuttle ships no -# JS wrapper). +# mapping and grab the :99 display in-container) + bash (the entrypoint script). +# No nodejs, no python (the daemon is a static Go binary). RUN apt-get update && apt-get install -y --no-install-recommends \ libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 \ libdbus-1-3 libdrm2 libxkbcommon0 libatspi2.0-0 libxcomposite1 \ @@ -31,24 +47,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ fonts-tlwg-loma-otf \ fontconfig fonts-liberation2 fonts-crosextra-carlito fonts-crosextra-caladea \ xvfb xdotool openbox \ - x11-utils scrot \ + x11-utils scrot bash \ curl ca-certificates xz-utils \ && rm -rf /var/lib/apt/lists/* WORKDIR /app -# Runtime deps + the container-only `server` group (geoip/proxy, which the -# published CLI does not need) + fonttools for the font-pack step, from the -# lockfile (reproducible). Keyed on pyproject/uv.lock ONLY, so editing the -# Python sources below never re-resolves deps or re-downloads the engines. -# No binary prebake: the fork binary is baked below and CLOAKBROWSER_BINARY_PATH -# bypasses any download path. -COPY --from=ghcr.io/astral-sh/uv:0.11.27 /uv /usr/local/bin/uv -COPY pyproject.toml uv.lock ./ -RUN uv export --frozen --no-default-groups --group server --group build --no-emit-project --no-hashes -o /tmp/req.txt \ - && uv pip install --system --no-cache -r /tmp/req.txt \ - && rm /tmp/req.txt - # --- Browser engine (baked prebuilt, selected by CLOAKBROWSER_BINARY_PATH). --- # Two free stealth-Chromium forks, both linux-x64 (this image is amd64 only). # clark is the default; clearcote the fallback. No proprietary binary. @@ -95,7 +99,7 @@ RUN mkdir -p /opt/clearcote \ # but only runs when CUTTLE_VNC=1 - the entrypoint then starts Xvnc instead of # Xvfb, so the default (prod) image is unaffected. No apt repo upstream; the # GitHub .deb is pinned. Base is trixie, so the trixie build matches. -# Pinned to the CloakBrowser Manager-proven combo: KasmVNC 1.3.3 (bookworm +# Pinned to a proven combo: KasmVNC 1.3.3 (bookworm # .deb, installs cleanly on trixie) + stock noVNC 1.5.x client. Do NOT bump # either independently: KasmVNC speaks a forked RFB dialect (non-standard # PointerEvent wire format, own message types) and newer stock noVNC (1.7) @@ -117,50 +121,32 @@ ARG NOVNC_VERSION=1.5.0 RUN mkdir -p /opt/cuttle-www \ && curl -fsSL "https://github.com/novnc/noVNC/archive/refs/tags/v${NOVNC_VERSION}.tar.gz" \ | tar xz -C /opt/cuttle-www --strip-components=1 "noVNC-${NOVNC_VERSION}/core" "noVNC-${NOVNC_VERSION}/vendor" -COPY bin/vnc-viewer.html /opt/cuttle-www/index.html +COPY ops/docker/bin/vnc-viewer.html /opt/cuttle-www/index.html # --- Windows font pack --- -# Some anti-bot JS font-enumerates for Windows families; a Windows-claiming -# fingerprint must present real family NAMES. Provide them via metric-compatible -# free fonts (Liberation, Carlito, Caladea) renamed to the Windows names - all -# from Debian main, no proprietary download. cuttleserve passes +# Pre-baked metric-compatible free fonts (Liberation/Carlito/Caladea) whose name +# table reports the corresponding Windows family, committed under winfonts/ (see +# winfonts/README.md for provenance). No Microsoft font software is included. +# Here they are only registered with fontconfig; cuttleserve passes # --fingerprint-fonts-dir=/opt/winfonts for the fork binaries. -COPY scripts/rename-fonts.py /tmp/rename-fonts.py -RUN set -e; \ - mkdir -p /opt/winfonts; \ - L="$(dirname "$(fc-list | grep -m1 -i LiberationSans-Regular | cut -d: -f1)")"; \ - X="$(dirname "$(fc-list | grep -m1 -i Carlito-Regular | cut -d: -f1)")"; \ - for s in Regular Bold Italic BoldItalic; do \ - python3 /tmp/rename-fonts.py "$L/LiberationSans-$s.ttf" "Arial" "/opt/winfonts/arial-$s.ttf"; \ - python3 /tmp/rename-fonts.py "$L/LiberationSerif-$s.ttf" "Times New Roman" "/opt/winfonts/times-$s.ttf"; \ - python3 /tmp/rename-fonts.py "$L/LiberationMono-$s.ttf" "Courier New" "/opt/winfonts/cour-$s.ttf"; \ - python3 /tmp/rename-fonts.py "$X/Carlito-$s.ttf" "Calibri" "/opt/winfonts/calibri-$s.ttf"; \ - python3 /tmp/rename-fonts.py "$X/Carlito-$s.ttf" "Segoe UI" "/opt/winfonts/segoeui-$s.ttf"; \ - python3 /tmp/rename-fonts.py "$X/Caladea-$s.ttf" "Cambria" "/opt/winfonts/cambria-$s.ttf"; \ - done; \ - rm -f /tmp/rename-fonts.py; \ - printf '\n\n/opt/winfonts\n' > /etc/fonts/conf.d/00-winfonts.conf; \ - fc-cache -f; \ - echo "winfonts families:"; fc-scan --format='%{family[0]}\n' /opt/winfonts/*.ttf | sort -u - -# The vendored MIT multiplexer package (argument-builders + geoip + config) and -# the authored host CLI. Last of the Python layers on purpose: a source edit -# here reuses every cached layer above (deps, engines, KasmVNC, noVNC, fonts). -COPY README.md LICENSE THIRD-PARTY.md ./ -COPY vendor/ vendor/ -COPY cuttle/ cuttle/ -RUN uv pip install --system --no-cache --no-deps . +COPY ops/docker/winfonts/*.ttf /opt/winfonts/ +RUN printf '\n\n/opt/winfonts\n' > /etc/fonts/conf.d/00-winfonts.conf \ + && fc-cache -f \ + && echo "winfonts families:" && fc-scan --format='%{family[0]}\n' /opt/winfonts/*.ttf | sort -u -# The patched multiplexer: strips inline proxy creds and answers the proxy 407 -# over CDP (Fetch.continueWithAuth) so the fork binaries can use an authenticated -# residential proxy; stamps a synthetic browserContextId on service_worker CDP -# targets so playwright-core does not crash on them; replicates the fork's launch -# flag set (UA, canvas/rects noise, brand/platform-version, Windows fonts dir). -COPY bin/cuttleserve /usr/local/bin/cuttleserve -RUN chmod +x /usr/local/bin/cuttleserve && python3 -m py_compile /usr/local/bin/cuttleserve +# The static Go daemon. `cuttle serve` is the multiplexer: it strips inline proxy +# creds and answers the proxy 407 over CDP (Fetch.continueWithAuth) so the fork +# binaries can use an authenticated residential proxy; stamps a synthetic +# browserContextId on service_worker CDP targets so CDP clients do not crash on +# them; and replicates the fork's launch flag set (UA, canvas/rects noise, +# brand/platform-version, Windows fonts dir). `cuttleserve` is a compatibility +# shim so the host CLI's `docker run ... cuttleserve` invocation keeps working. +COPY --from=builder /out/cuttle /usr/local/bin/cuttle +RUN printf '#!/bin/sh\nexec cuttle serve "$@"\n' > /usr/local/bin/cuttleserve \ + && chmod +x /usr/local/bin/cuttleserve # Headed-mode entrypoint (Xvfb + openbox), then the user command. -COPY bin/docker-entrypoint.sh /entrypoint.sh +COPY ops/docker/bin/docker-entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh # 9222: CDP (agent). 6080: KasmVNC web viewer (only served when CUTTLE_VNC=1). diff --git a/ops/docker/Dockerfile.dockerignore b/ops/docker/Dockerfile.dockerignore new file mode 100644 index 0000000..2471182 --- /dev/null +++ b/ops/docker/Dockerfile.dockerignore @@ -0,0 +1,17 @@ +.git +.github +docs +ops +!ops/docker +test +.playwright-cli +dist +*.test +coverage.out +/cuttle +*.md +.golangci.yml +lefthook.yml +Justfile +.gitignore +.env diff --git a/bin/docker-entrypoint.sh b/ops/docker/bin/docker-entrypoint.sh similarity index 96% rename from bin/docker-entrypoint.sh rename to ops/docker/bin/docker-entrypoint.sh index 33b1789..10070f0 100755 --- a/bin/docker-entrypoint.sh +++ b/ops/docker/bin/docker-entrypoint.sh @@ -17,8 +17,8 @@ rm -f /tmp/.X99-lock /tmp/.X11-unix/X99 # on Xvnc: the loopback port mapping (run with -p 127.0.0.1:PORT:PORT) is the # security boundary. Downstream (openbox, xdotool, Chromium) only needs :99 up. # -# In VNC mode we also make the browser presentable for a human viewer (args -# match CloakBrowser Manager's launch): a bare positional URL, which cuttleserve +# In VNC mode we also make the browser presentable for a human viewer (a headed +# viewer launch): a bare positional URL, which cuttleserve # passes through to Chrome's argv, so headed Chrome maps a visible top-level # window (a pure CDP-scraping launch is windowless); --start-maximized so # openbox sizes it to the full display; --test-type to suppress the "unsupported diff --git a/bin/vnc-viewer.html b/ops/docker/bin/vnc-viewer.html similarity index 98% rename from bin/vnc-viewer.html rename to ops/docker/bin/vnc-viewer.html index d76be72..e7d5b0c 100644 --- a/bin/vnc-viewer.html +++ b/ops/docker/bin/vnc-viewer.html @@ -21,7 +21,7 @@ // KasmVNC speaks a forked RFB dialect: PointerEvent is 11 bytes (u16 button // mask + s16 scroll pair) instead of the standard 6. Sending the standard // form desyncs the server's stream parser and it drops the connection on the -// first mouse move. Same rewrite CloakBrowser Manager does in its ws proxy. +// first mouse move, so the ws proxy rewrites it to KasmVNC's wire format. RFB.messages.pointerEvent = (sock, x, y, mask) => { sock.sQpush8(5); sock.sQpush16(mask); diff --git a/ops/docker/winfonts/README.md b/ops/docker/winfonts/README.md new file mode 100644 index 0000000..a390321 --- /dev/null +++ b/ops/docker/winfonts/README.md @@ -0,0 +1,25 @@ +# winfonts + +Metric-compatible free fonts whose internal `name` table has been set to report +the corresponding Windows family. A Windows-claiming fingerprint is expected to +expose these family names, and some anti-bot JS enumerates fonts by measuring +text width, so the substitutes must render at Windows metrics and report the +Windows family name. + +**No Microsoft font software is included here.** These are free, redistributable +fonts with only their family name rewritten: + +| Windows family reported | Free font used | License | +| ----------------------- | ------------------------- | -------------------- | +| Arial | Liberation Sans | OFL 1.1 | +| Times New Roman | Liberation Serif | OFL 1.1 | +| Courier New | Liberation Mono | OFL 1.1 | +| Calibri, Segoe UI | Carlito | OFL 1.1 | +| Cambria | Caladea | OFL 1.1 | + +The Dockerfile copies these into `/opt/winfonts` and registers them with +fontconfig; `cuttle serve` passes `--fingerprint-fonts-dir=/opt/winfonts` to the +fork binaries. To regenerate from upstream Debian packages, install +`fonts-liberation2 fonts-crosextra-carlito fonts-crosextra-caladea` and rewrite +the `name` table records (family/full/postscript/typographic) to the target +family. diff --git a/ops/docker/winfonts/arial-Bold.ttf b/ops/docker/winfonts/arial-Bold.ttf new file mode 100644 index 0000000..8865fbc Binary files /dev/null and b/ops/docker/winfonts/arial-Bold.ttf differ diff --git a/ops/docker/winfonts/arial-BoldItalic.ttf b/ops/docker/winfonts/arial-BoldItalic.ttf new file mode 100644 index 0000000..7b4d033 Binary files /dev/null and b/ops/docker/winfonts/arial-BoldItalic.ttf differ diff --git a/ops/docker/winfonts/arial-Italic.ttf b/ops/docker/winfonts/arial-Italic.ttf new file mode 100644 index 0000000..8774224 Binary files /dev/null and b/ops/docker/winfonts/arial-Italic.ttf differ diff --git a/ops/docker/winfonts/arial-Regular.ttf b/ops/docker/winfonts/arial-Regular.ttf new file mode 100644 index 0000000..f2489da Binary files /dev/null and b/ops/docker/winfonts/arial-Regular.ttf differ diff --git a/ops/docker/winfonts/calibri-Bold.ttf b/ops/docker/winfonts/calibri-Bold.ttf new file mode 100644 index 0000000..77b9732 Binary files /dev/null and b/ops/docker/winfonts/calibri-Bold.ttf differ diff --git a/ops/docker/winfonts/calibri-BoldItalic.ttf b/ops/docker/winfonts/calibri-BoldItalic.ttf new file mode 100644 index 0000000..0640432 Binary files /dev/null and b/ops/docker/winfonts/calibri-BoldItalic.ttf differ diff --git a/ops/docker/winfonts/calibri-Italic.ttf b/ops/docker/winfonts/calibri-Italic.ttf new file mode 100644 index 0000000..bc811fd Binary files /dev/null and b/ops/docker/winfonts/calibri-Italic.ttf differ diff --git a/ops/docker/winfonts/calibri-Regular.ttf b/ops/docker/winfonts/calibri-Regular.ttf new file mode 100644 index 0000000..25f9a56 Binary files /dev/null and b/ops/docker/winfonts/calibri-Regular.ttf differ diff --git a/ops/docker/winfonts/cambria-Bold.ttf b/ops/docker/winfonts/cambria-Bold.ttf new file mode 100644 index 0000000..40fe958 Binary files /dev/null and b/ops/docker/winfonts/cambria-Bold.ttf differ diff --git a/ops/docker/winfonts/cambria-BoldItalic.ttf b/ops/docker/winfonts/cambria-BoldItalic.ttf new file mode 100644 index 0000000..1d31a9e Binary files /dev/null and b/ops/docker/winfonts/cambria-BoldItalic.ttf differ diff --git a/ops/docker/winfonts/cambria-Italic.ttf b/ops/docker/winfonts/cambria-Italic.ttf new file mode 100644 index 0000000..5e9ae2f Binary files /dev/null and b/ops/docker/winfonts/cambria-Italic.ttf differ diff --git a/ops/docker/winfonts/cambria-Regular.ttf b/ops/docker/winfonts/cambria-Regular.ttf new file mode 100644 index 0000000..ab90a95 Binary files /dev/null and b/ops/docker/winfonts/cambria-Regular.ttf differ diff --git a/ops/docker/winfonts/cour-Bold.ttf b/ops/docker/winfonts/cour-Bold.ttf new file mode 100644 index 0000000..3b77fb5 Binary files /dev/null and b/ops/docker/winfonts/cour-Bold.ttf differ diff --git a/ops/docker/winfonts/cour-BoldItalic.ttf b/ops/docker/winfonts/cour-BoldItalic.ttf new file mode 100644 index 0000000..cf725e7 Binary files /dev/null and b/ops/docker/winfonts/cour-BoldItalic.ttf differ diff --git a/ops/docker/winfonts/cour-Italic.ttf b/ops/docker/winfonts/cour-Italic.ttf new file mode 100644 index 0000000..047915e Binary files /dev/null and b/ops/docker/winfonts/cour-Italic.ttf differ diff --git a/ops/docker/winfonts/cour-Regular.ttf b/ops/docker/winfonts/cour-Regular.ttf new file mode 100644 index 0000000..d1fd4ce Binary files /dev/null and b/ops/docker/winfonts/cour-Regular.ttf differ diff --git a/ops/docker/winfonts/segoeui-Bold.ttf b/ops/docker/winfonts/segoeui-Bold.ttf new file mode 100644 index 0000000..054a350 Binary files /dev/null and b/ops/docker/winfonts/segoeui-Bold.ttf differ diff --git a/ops/docker/winfonts/segoeui-BoldItalic.ttf b/ops/docker/winfonts/segoeui-BoldItalic.ttf new file mode 100644 index 0000000..245b8a6 Binary files /dev/null and b/ops/docker/winfonts/segoeui-BoldItalic.ttf differ diff --git a/ops/docker/winfonts/segoeui-Italic.ttf b/ops/docker/winfonts/segoeui-Italic.ttf new file mode 100644 index 0000000..2f7bd27 Binary files /dev/null and b/ops/docker/winfonts/segoeui-Italic.ttf differ diff --git a/ops/docker/winfonts/segoeui-Regular.ttf b/ops/docker/winfonts/segoeui-Regular.ttf new file mode 100644 index 0000000..935e872 Binary files /dev/null and b/ops/docker/winfonts/segoeui-Regular.ttf differ diff --git a/ops/docker/winfonts/times-Bold.ttf b/ops/docker/winfonts/times-Bold.ttf new file mode 100644 index 0000000..21d6989 Binary files /dev/null and b/ops/docker/winfonts/times-Bold.ttf differ diff --git a/ops/docker/winfonts/times-BoldItalic.ttf b/ops/docker/winfonts/times-BoldItalic.ttf new file mode 100644 index 0000000..c9763ed Binary files /dev/null and b/ops/docker/winfonts/times-BoldItalic.ttf differ diff --git a/ops/docker/winfonts/times-Italic.ttf b/ops/docker/winfonts/times-Italic.ttf new file mode 100644 index 0000000..4e42c75 Binary files /dev/null and b/ops/docker/winfonts/times-Italic.ttf differ diff --git a/ops/docker/winfonts/times-Regular.ttf b/ops/docker/winfonts/times-Regular.ttf new file mode 100644 index 0000000..82d6786 Binary files /dev/null and b/ops/docker/winfonts/times-Regular.ttf differ diff --git a/ops/helm/cuttle/.helmignore b/ops/helm/cuttle/.helmignore new file mode 100644 index 0000000..4bb8985 --- /dev/null +++ b/ops/helm/cuttle/.helmignore @@ -0,0 +1,5 @@ +.git +*.tgz +*.orig +*.swp +.DS_Store diff --git a/ops/helm/cuttle/Chart.yaml b/ops/helm/cuttle/Chart.yaml new file mode 100644 index 0000000..b8c5f01 --- /dev/null +++ b/ops/helm/cuttle/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: cuttle +description: Stealth-Chromium CDP farm - one Deployment running `cuttle serve`, reached over kubectl port-forward. +type: application +# Chart version tracks its own shape; appVersion is overridden per-release via +# values.image.tag (pinned to the CLI version by `cuttle up`). +version: 0.1.0 +appVersion: "0.3.0" +kubeVersion: ">=1.21.0-0" +home: https://glim.sh +sources: + - https://github.com/glim-sh/cuttle +keywords: + - chromium + - cdp + - browser +maintainers: + - name: glim.sh diff --git a/ops/helm/cuttle/templates/NOTES.txt b/ops/helm/cuttle/templates/NOTES.txt new file mode 100644 index 0000000..6763d58 --- /dev/null +++ b/ops/helm/cuttle/templates/NOTES.txt @@ -0,0 +1,10 @@ +cuttle is starting in namespace {{ .Release.Namespace }} (release {{ .Release.Name }}). + +Profile storage: {{ .Values.profileStorage }}{{ if eq .Values.profileStorage "remote" }} (durable PVC, kept across uninstall){{ else }} (ephemeral emptyDir; auth state is checked out/in over CDP){{ end }}. + +The Service is ClusterIP-only with a deny-all-ingress NetworkPolicy. Reach it +over a port-forward (this is what `cuttle` does for a k8s context): + + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "cuttle.fullname" . }} 9222:9222{{ if .Values.vnc }} 6080:6080{{ end }} + +Then point a CDP client at http://127.0.0.1:9222?fingerprint=. diff --git a/ops/helm/cuttle/templates/_helpers.tpl b/ops/helm/cuttle/templates/_helpers.tpl new file mode 100644 index 0000000..b8a57a3 --- /dev/null +++ b/ops/helm/cuttle/templates/_helpers.tpl @@ -0,0 +1,31 @@ +{{- define "cuttle.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "cuttle.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "cuttle.labels" -}} +app.kubernetes.io/name: {{ include "cuttle.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- end -}} + +{{- define "cuttle.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cuttle.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/ops/helm/cuttle/templates/deployment.yaml b/ops/helm/cuttle/templates/deployment.yaml new file mode 100644 index 0000000..8032b70 --- /dev/null +++ b/ops/helm/cuttle/templates/deployment.yaml @@ -0,0 +1,98 @@ +{{- $remote := eq .Values.profileStorage "remote" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cuttle.fullname" . }} + labels: + {{- include "cuttle.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + # A remote-persistent PVC is RWO: it cannot mount into a new pod while the old + # one still holds it, so recreate rather than surge. emptyDir has no such + # constraint, but a single-writer CDP browser must never run two pods anyway. + type: Recreate + selector: + matchLabels: + {{- include "cuttle.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "cuttle.selectorLabels" . | nindent 8 }} + spec: + nodeSelector: + # The image is linux/amd64 only (clark/clearcote prebuilts); pin hard so + # it never schedules on arm. Merged after operator selectors so this wins. + {{- with .Values.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + kubernetes.io/arch: amd64 + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cuttle + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + # The image ENTRYPOINT bootstraps the X server (headed Chrome is + # required to clear escalated anti-bot challenges); args are what it + # execs. Do not set `command` - that would bypass the X bootstrap. + args: + {{- if $remote }} + # Clear a stale Chrome SingletonLock left by a crashed pod before it + # can wedge a fresh start on the reused PVC, then run the daemon. + - sh + - -c + - >- + find {{ .Values.dataDir }} -name 'Singleton*' -delete 2>/dev/null || true; + exec cuttle serve --headless=false --data-dir={{ .Values.dataDir }} --keep-profile + {{- else }} + - cuttle + - serve + - --headless=false + - --data-dir={{ .Values.dataDir }} + {{- end }} + env: + - name: CUTTLESERVE_HOST + value: "0.0.0.0" + {{- with .Values.proxy }} + - name: CUTTLESERVE_PROXY + value: {{ . | quote }} + {{- end }} + {{- if .Values.vnc }} + - name: CUTTLE_VNC + value: "1" + - name: CUTTLE_VNC_PORT + value: "6080" + {{- end }} + ports: + - name: cdp + containerPort: 9222 + protocol: TCP + {{- if .Values.vnc }} + - name: vnc + containerPort: 6080 + protocol: TCP + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: {{ .Values.dataDir }} + # Chrome needs a large /dev/shm or it crashes under load; the default + # 64Mi tmpfs is not enough. + - name: dshm + mountPath: /dev/shm + volumes: + - name: data + {{- if $remote }} + persistentVolumeClaim: + claimName: {{ include "cuttle.fullname" . }}-data + {{- else }} + emptyDir: {} + {{- end }} + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 2Gi diff --git a/ops/helm/cuttle/templates/networkpolicy.yaml b/ops/helm/cuttle/templates/networkpolicy.yaml new file mode 100644 index 0000000..a6409df --- /dev/null +++ b/ops/helm/cuttle/templates/networkpolicy.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "cuttle.fullname" . }}-deny-ingress + labels: + {{- include "cuttle.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "cuttle.selectorLabels" . | nindent 6 }} + # Deny all ingress: CDP is unauthenticated, and `kubectl port-forward` reaches + # the pod directly (it does not traverse the Service network path), so no + # in-cluster ingress is needed. Egress stays open (Chrome must reach the web). + policyTypes: + - Ingress + ingress: [] diff --git a/ops/helm/cuttle/templates/pvc.yaml b/ops/helm/cuttle/templates/pvc.yaml new file mode 100644 index 0000000..c3e431a --- /dev/null +++ b/ops/helm/cuttle/templates/pvc.yaml @@ -0,0 +1,21 @@ +{{- if eq .Values.profileStorage "remote" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "cuttle.fullname" . }}-data + labels: + {{- include "cuttle.labels" . | nindent 4 }} + annotations: + # Survive `helm uninstall` so a re-install keeps the persisted profiles; + # remove it explicitly (`cuttle down --purge` deletes the PVC). + helm.sh/resource-policy: keep +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- with .Values.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/ops/helm/cuttle/templates/service.yaml b/ops/helm/cuttle/templates/service.yaml new file mode 100644 index 0000000..69c1807 --- /dev/null +++ b/ops/helm/cuttle/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cuttle.fullname" . }} + labels: + {{- include "cuttle.labels" . | nindent 4 }} +spec: + # ClusterIP only - reached via `kubectl port-forward`, never exposed. CDP has + # no auth; anyone who can reach it drives your logged-in browser. + type: ClusterIP + selector: + {{- include "cuttle.selectorLabels" . | nindent 4 }} + ports: + - name: cdp + port: 9222 + targetPort: cdp + protocol: TCP + {{- if .Values.vnc }} + - name: vnc + port: 6080 + targetPort: vnc + protocol: TCP + {{- end }} diff --git a/ops/helm/cuttle/values.yaml b/ops/helm/cuttle/values.yaml new file mode 100644 index 0000000..ca6d51b --- /dev/null +++ b/ops/helm/cuttle/values.yaml @@ -0,0 +1,50 @@ +# cuttle Helm chart values. The CLI (`cuttle up` with a k8s context) shells +# `helm upgrade --install` against this chart and sets image.tag/proxy/etc from +# the context config; these are the defaults for a manual `helm install`. + +image: + repository: ghcr.io/glim-sh/cuttle + # Pin to the CLI version so cuttleserve never skews from the client. + tag: "0.3.0" + pullPolicy: IfNotPresent + +# One browser per release. CDP is single-writer; do not scale this up. +replicaCount: 1 + +# profileStorage selects the profile backing store: +# local - ephemeral emptyDir scratch; auth state is checked out/in over CDP +# from the operator's machine (local-canonical, nothing persists here). +# remote - a durable PVC (RWO) for autonomous / always-on use where the +# operator's machine is not present to inject state. +profileStorage: local + +# Where the per-seed Chrome user_data_dirs live inside the container. The scratch +# volume (emptyDir or PVC) mounts here. +dataDir: /data + +# Default proxy applied to every seed at browser startup (CUTTLESERVE_PROXY). +# geoip (tz/locale/exit-IP) auto-follows it. Empty = no default proxy. +proxy: "" + +# The image is linux/amd64 only (clark/clearcote ship linux-x64 prebuilts); a +# hard kubernetes.io/arch=amd64 selector is always merged in - never schedule on +# arm. Extra selectors here are additive (e.g. a dedicated browser node pool). +nodeSelector: {} + +tolerations: [] + +resources: + requests: + memory: 1Gi + cpu: 500m + limits: + memory: 4Gi + +# PVC settings, used only when profileStorage=remote. +persistence: + size: 5Gi + # Leave null to use the cluster default StorageClass. + storageClass: "" + +# CUTTLE_VNC=1 makes the KasmVNC web viewer on 6080 live for login handoff. +vnc: true diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 6c8274c..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,81 +0,0 @@ -[build-system] -requires = ["setuptools>=68"] -build-backend = "setuptools.build_meta" - -[project] -name = "cuttle-browser" -version = "0.3.0" -description = "Stealth-Chromium CDP farm: per-seed browser fingerprints over a patched CDP multiplexer." -readme = "README.md" -license = { text = "MIT" } -requires-python = "==3.12.*" -dependencies = [ - "aiohttp>=3.9", - "websockets>=12.0", -] - -[dependency-groups] -dev = [ - "ruff>=0.15.0", - "ty>=0.0.30", -] -# Container-only: cuttleserve resolves geoip and talks to proxies via the -# vendored cloakbrowser.geoip (which imports these lazily). bin/cuttleserve is -# not part of the wheel, so the published CLI never reaches that code path - -# keeping these out of [project.dependencies] halves what pip/brew/nix install. -server = [ - "httpx>=0.27", - "geoip2>=4.0", - "socksio>=1.0", -] -# Build-time only: fonttools renames the Windows font pack during the image -# build (scripts/rename-fonts.py). Not a runtime dep and not a dev tool, so it -# gets its own group - the Dockerfile installs it from the lock, nothing else does. -build = [ - "fonttools>=4.0", -] - -[project.scripts] -cuttle = "cuttle.cli:main" - -[tool.setuptools] -packages = ["cloakbrowser", "cuttle"] - -# The vendored upstream subset lives under vendor/ but must import as the -# upstream `cloakbrowser` package (so bin/cuttleserve's imports and vendor-sync -# diffs stay verbatim). `cuttle` is our authored package at the repo root. -[tool.setuptools.package-dir] -cloakbrowser = "vendor/cloakbrowser" -cuttle = "cuttle" - -# Bundle SKILL.md into the wheel so `cuttle skill` works from a plain pip install -# (no repo present). The file is `cuttle/SKILL.md`, a symlink to the canonical -# root SKILL.md - the build resolves it to real content in the wheel. -[tool.setuptools.package-data] -cuttle = ["SKILL.md"] - -# ruff/ty only touch AUTHORED code. vendor/cloakbrowser and bin/cuttleserve are -# vendored/patched from upstream cloakbrowser - reformatting or retyping them -# would break the "verbatim" provenance and blow up scripts/sync.sh diffs. -[tool.ruff] -target-version = "py312" -line-length = 100 -extend-exclude = ["vendor"] - -[tool.ruff.lint] -select = ["E", "F", "I", "UP"] -ignore = ["E501", "E741", "UP007"] - -[tool.ruff.format] -quote-style = "double" -indent-style = "space" -line-ending = "lf" - -[tool.ty.environment] -python-version = "3.12" - -# scripts/rename-fonts.py is a build-time script whose only import (fontTools) is -# installed in the image, not the project env; it gets ruff formatting but not -# type-checking. test/ and cuttle/ are the authored app code worth typing. -[tool.ty.src] -include = ["test", "cuttle"] diff --git a/scripts/check-skill-version.py b/scripts/check-skill-version.py deleted file mode 100644 index fc2c5aa..0000000 --- a/scripts/check-skill-version.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -"""Check (or with --fix, set) SKILL.md's frontmatter version against the package. - -The `cuttle up` briefing prints the installed cuttle-browser version so an -agent holding a stale installed copy of the skill can notice and rerun -`cuttle skill`. That detection only works if SKILL.md's metadata.version is -kept in lockstep with [project].version - this check enforces it. release-please -bumps both together in the release PR (SKILL.md via its `x-release-please-version` -frontmatter annotation); --fix is the manual fallback if they ever drift. -""" - -from __future__ import annotations - -import re -import sys -import tomllib -from pathlib import Path - -VERSION_RE = re.compile(r'^(\s*version:\s*)"([^"]+)"', re.M) - -root = Path(__file__).resolve().parent.parent -skill_md = root / "SKILL.md" -pkg = tomllib.loads((root / "pyproject.toml").read_text())["project"]["version"] - -text = skill_md.read_text() -if "--fix" in sys.argv: - text, n = VERSION_RE.subn(rf'\g<1>"{pkg}"', text, count=1) - if not n: - sys.exit("SKILL.md has no metadata.version line to set") - skill_md.write_text(text) - -m = VERSION_RE.search(text) -skill = m.group(2) if m else None -if skill != pkg: - sys.exit( - f"SKILL.md metadata.version {skill!r} != pyproject [project].version {pkg!r}" - " - bump them together (the briefing's stale-skill detection depends on it)" - ) -print(f"skill version in lockstep ({pkg})") diff --git a/scripts/rename-fonts.py b/scripts/rename-fonts.py deleted file mode 100644 index ede214a..0000000 --- a/scripts/rename-fonts.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -"""Rename a font's family so metric-compatible free fonts present the Windows -family names a Windows fingerprint profile is expected to expose -(Calibri<-Carlito, Cambria<-Caladea, "Segoe UI"<-Carlito). Rewrites the name -table family/full/postscript/typographic records to the target family.""" - -import sys - -from fontTools.ttLib import TTFont - -src, target, out = sys.argv[1], sys.argv[2], sys.argv[3] -ps = target.replace(" ", "") -font = TTFont(src) -name = font["name"] -for rec in name.names: - if rec.nameID in (1, 16): # family / typographic family - rec.string = target - elif rec.nameID == 4: # full name - rec.string = target - elif rec.nameID == 6: # postscript name - rec.string = ps -font.save(out) -print(f"{src} -> {target} ({out})") diff --git a/scripts/render-homebrew-formula.py b/scripts/render-homebrew-formula.py deleted file mode 100644 index 52f3979..0000000 --- a/scripts/render-homebrew-formula.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -"""Render the homebrew tap formula (Formula/cuttle.rb) from uv.lock. - -The formula installs the released sdist into a keg virtualenv; every runtime -dependency is pinned to the exact sdist uv.lock resolved. Only [project.dependencies] -is walked - the container-only `server` group stays out of the formula, which is -also why no dep here needs a rust toolchain to build from sdist. Stdlib only - CI -runs it with plain python3. -""" - -import argparse -import hashlib -import sys -import tomllib -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent - -TEMPLATE = """\ -# typed: false -# frozen_string_literal: true - -# Generated by glim-sh/cuttle scripts/render-homebrew-formula.py - do not edit. -class Cuttle < Formula - include Language::Python::Virtualenv - - desc "Host CLI for the cuttle stealth-Chromium CDP farm" - homepage "https://github.com/glim-sh/cuttle" - url "https://github.com/glim-sh/cuttle/releases/download/v@VERSION@/cuttle_browser-@VERSION@.tar.gz" - sha256 "@SHA256@" - license "MIT" - - livecheck do - url :stable - strategy :github_releases - regex(/^v?(\\d+(?:\\.\\d+)+)$/i) - end - - depends_on "python@3.12" - -@RESOURCES@ - - def install - virtualenv_install_with_resources - end - - test do - assert_match "usage: cuttle", shell_output("#{bin}/cuttle --help") - end -end -""" - - -def runtime_closure(lock: dict) -> list[str]: - pkgs = {p["name"]: p for p in lock["package"]} - closure: set[str] = set() - queue = [d["name"] for d in pkgs["cuttle-browser"].get("dependencies", [])] - while queue: - name = queue.pop() - if name in closure: - continue - closure.add(name) - queue += [d["name"] for d in pkgs[name].get("dependencies", [])] - return sorted(closure) - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--version", required=True) - ap.add_argument("--sdist", required=True, type=Path, help="released sdist, for its sha256") - ap.add_argument("--out", required=True, type=Path) - args = ap.parse_args() - - lock = tomllib.loads((ROOT / "uv.lock").read_text()) - pkgs = {p["name"]: p for p in lock["package"]} - - resources = [] - for name in runtime_closure(lock): - sdist = pkgs[name].get("sdist") - if sdist is None: - print(f"error: {name} has no sdist in uv.lock (wheel-only?)", file=sys.stderr) - return 1 - sha = sdist["hash"].removeprefix("sha256:") - resources.append( - f' resource "{name}" do\n url "{sdist["url"]}"\n sha256 "{sha}"\n end' - ) - - formula = ( - TEMPLATE.replace("@VERSION@", args.version) - .replace("@SHA256@", hashlib.sha256(args.sdist.read_bytes()).hexdigest()) - .replace("@RESOURCES@", "\n\n".join(resources)) - ) - args.out.write_text(formula) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/sync.sh b/scripts/sync.sh deleted file mode 100755 index f2f471a..0000000 --- a/scripts/sync.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Re-sync the vendored cloakbrowser subset for review. This does NOT overwrite -# the vendored files (they carry deliberate trims/stubs - see docs/UPSTREAM.md). -# It fetches the pinned upstream files into a temp dir and prints a diff against -# ours, so an upstream bump is a reviewable change you re-apply by hand. -set -euo pipefail - -REF="${CUTTLE_UPSTREAM_REF:-v0.4.9}" -REPO="${CUTTLE_UPSTREAM_REPO:-https://github.com/CloakHQ/cloakbrowser}" -HERE="$(cd "$(dirname "$0")/.." && pwd)" -TMP="$(mktemp -d)" -trap 'rm -rf "$TMP"' EXIT - -echo "Fetching $REPO @ $REF ..." -git clone --quiet --depth 1 --branch "$REF" "$REPO" "$TMP/upstream" - -for f in config.py geoip.py browser.py download.py; do - echo "" - echo "===== vendor/cloakbrowser/$f vs upstream cloakbrowser/$f =====" - diff -u "$TMP/upstream/cloakbrowser/$f" "$HERE/vendor/cloakbrowser/$f" || true -done - -echo "" -echo "===== bin/cuttleserve vs upstream bin/cloakserve =====" -diff -u "$TMP/upstream/bin/cloakserve" "$HERE/bin/cuttleserve" || true - -echo "" -echo "Review the diffs above, re-apply cuttle's trims/patches as needed, then" -echo "update the pinned ref in docs/UPSTREAM.md." diff --git a/test/README.md b/test/README.md deleted file mode 100644 index 001c3e1..0000000 --- a/test/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# cuttle smoke harness - -A neutral, self-contained smoke that drives a running cuttle over CDP and -introspects each seed's browser directly - no third-party sites, no network -targets, no local server. Raw CDP over `websockets` (the dependency cuttle -already ships), so it stays pure-Python with no browser-automation toolchain. - -## What it checks - -1. **per-seed fingerprint isolation.** Each fingerprint seed gets its own - coherent identity, so an in-page canvas readback differs across seeds. -2. **stealth coherence.** `navigator.webdriver` is falsy and the UA/platform - agree (a Windows UA must not pair with a non-Windows platform). -3. **connection stability under cold-cycle load.** Fresh seeds are launched in a - loop; every cycle must connect and probe without error. - -Everything is read in-page on an `about:blank` target - nothing leaves the harness. - -## Run - -From the repo root (`websockets` is a declared dependency, so the project env has it): - -```bash -uv sync -CUTTLE_URL=http://127.0.0.1:9222 uv run python test/harness.py && echo GREEN -``` - -## Config - -| Env | Default | Meaning | -|-----|---------|---------| -| `CUTTLE_URL` | `http://127.0.0.1:9222` | cuttle CDP endpoint | -| `COLD_CYCLES` | `3` | Number of fresh-seed cold cycles | - -## Green means - -- Distinct canvas readbacks across seeds (per-seed fingerprint isolation works). -- Coherent stealth signals on every cycle (`navigator.webdriver` falsy, - UA/platform agree). -- Every cold cycle connects and probes without error. - -## Scope - -This is a fast, client-agnostic local smoke of the mechanical stealth path. It -deliberately does NOT reproduce the playwright-core service_worker crash (only a -playwright client can observe that), nor does it clear real challenges. Those -are validated separately against a real amd64 deployment - see -[../docs/UPGRADE.md](../docs/UPGRADE.md). diff --git a/test/harness.py b/test/harness.py deleted file mode 100644 index 1b6793a..0000000 --- a/test/harness.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python3 -"""cuttle smoke harness - neutral, self-contained. - -Drives a running cuttle over CDP and introspects each seed's browser directly - -no third-party sites, no network targets, no local server. It checks: - - 1. per-seed fingerprint isolation - each fingerprint seed gets its own coherent - identity, so an in-page canvas readback differs across seeds. - 2. stealth coherence - navigator.webdriver is falsy and the UA/platform agree - (a Windows UA must not pair with a non-Windows platform). - 3. connection stability under cold-cycle load - fresh seeds are launched in a - loop; every cycle must connect and probe without error. - -Run: uv run python test/harness.py (from the repo root; see test/README.md) -Green = distinct per-seed canvas, coherent stealth signals, no failures. - -This is a fast, client-agnostic local smoke. It does NOT reproduce the -playwright-core service_worker crash (only a playwright client can), nor does it -clear real challenges - both are validated separately against a real amd64 -deployment. See ../docs/UPGRADE.md. -""" - -import asyncio -import json -import os -import time -import urllib.request -from urllib.parse import urlencode, urlsplit, urlunsplit - -import websockets - -CUTTLE_URL = os.environ.get("CUTTLE_URL", "http://127.0.0.1:9222") -COLD_CYCLES = int(os.environ.get("COLD_CYCLES", "3")) -RUN_ID = format(int(time.time()), "x") - -CONNECT_TIMEOUT = 90 - -# One self-contained expression: build a canvas (farbling is fingerprint-seeded, -# so it differs per seed) and read the stealth signals, returned as JSON. -PROBE_JS = r""" -(() => { - let canvas = "missing"; - try { - const c = document.createElement("canvas"); - c.width = 200; c.height = 40; - const ctx = c.getContext("2d"); - ctx.textBaseline = "top"; - ctx.font = "16px Arial"; - ctx.fillStyle = "#f60"; ctx.fillRect(0, 0, 200, 40); - ctx.fillStyle = "#069"; ctx.fillText("cuttle-smoke", 2, 2); - canvas = c.toDataURL(); - } catch (e) { canvas = "canvas-error:" + e.message; } - return JSON.stringify({ - webdriver: navigator.webdriver, - ua: navigator.userAgent, - platform: navigator.platform, - canvas, - }); -})() -""" - - -def browser_ws_for_seed(seed): - """Ask cuttle for the seed's browser CDP WebSocket (this launches the seed).""" - parts = urlsplit(CUTTLE_URL) - url = urlunsplit( - (parts.scheme, parts.netloc, "/json/version", urlencode({"fingerprint": seed}), "") - ) - with urllib.request.urlopen(url, timeout=30) as r: - return json.load(r)["webSocketDebuggerUrl"] - - -class CDP: - """Minimal CDP client: send a command, return its result (skipping events).""" - - def __init__(self, ws): - self.ws = ws - self._id = 0 - - async def send(self, method, params=None, session_id=None): - self._id += 1 - mid = self._id - msg = {"id": mid, "method": method, "params": params or {}} - if session_id: - msg["sessionId"] = session_id - await self.ws.send(json.dumps(msg)) - while True: - resp = json.loads(await self.ws.recv()) - if resp.get("id") == mid: - if "error" in resp: - raise RuntimeError(f"{method}: {resp['error']}") - return resp.get("result", {}) - - -async def probe_seed(seed): - ws_url = browser_ws_for_seed(seed) - async with websockets.connect(ws_url, max_size=None, open_timeout=CONNECT_TIMEOUT) as ws: - cdp = CDP(ws) - target = await cdp.send("Target.createTarget", {"url": "about:blank"}) - target_id = target["targetId"] - attached = await cdp.send("Target.attachToTarget", {"targetId": target_id, "flatten": True}) - session_id = attached["sessionId"] - result = await cdp.send( - "Runtime.evaluate", - {"expression": PROBE_JS, "returnByValue": True, "awaitPromise": True}, - session_id=session_id, - ) - await cdp.send("Target.closeTarget", {"targetId": target_id}) - return json.loads(result["result"]["value"]) - - -results = [] # (name, status, detail) -canvas_by_seed = {} - - -async def cold_cycle(cycle): - seed = f"smoke-{RUN_ID}-{cycle}" - name = f"cold-cycle-{cycle}" - try: - info = await asyncio.wait_for(probe_seed(seed), timeout=CONNECT_TIMEOUT + 30) - except Exception as exc: # a connection/probe failure IS the stability signal - results.append((name, "fail", f"probe failed: {exc}")) - return - - ua, platform = info.get("ua", ""), info.get("platform", "") - canvas = info.get("canvas", "missing") - canvas_by_seed[seed] = canvas - - problems = [] - if info.get("webdriver"): - problems.append(f"webdriver={info['webdriver']}") - if ("windows" in ua.lower()) != platform.lower().startswith("win"): - problems.append(f"incoherent ua/platform (ua={ua[:40]!r} platform={platform!r})") - if not canvas.startswith("data:image"): - problems.append(f"canvas={canvas[:24]}") - - if problems: - results.append((name, "fail", "; ".join(problems))) - else: - results.append( - ( - name, - "pass", - f"webdriver={info.get('webdriver')} platform={platform} canvas=ok seed={seed}", - ) - ) - - -def check_canvas_isolation(): - values = [v for v in canvas_by_seed.values() if v.startswith("data:image")] - if len(values) < 2: - results.append( - ("canvas-isolation", "fail", f"need >=2 canvas readbacks, got {len(values)}") - ) - return - distinct = len(set(values)) - if distinct < 2: - results.append( - ( - "canvas-isolation", - "fail", - f"all {len(values)} seeds produced an identical canvas (no per-seed farbling)", - ) - ) - else: - results.append( - ( - "canvas-isolation", - "pass", - f"{distinct} distinct canvas fingerprints across {len(values)} seeds", - ) - ) - - -async def main(): - print("cuttle smoke harness") - print(f" CUTTLE_URL = {CUTTLE_URL}") - print(f" cold cycles = {COLD_CYCLES}") - - print("\n== stealth coherence + connection stability (cold cycles) ==") - for i in range(1, COLD_CYCLES + 1): - await cold_cycle(i) - - print("\n== per-seed fingerprint isolation ==") - check_canvas_isolation() - - for name, status, detail in results: - print(f" [{status.upper()}] {name} - {detail}") - - passed = sum(1 for _, s, _ in results if s == "pass") - print("\n================ SUMMARY ================") - print(f" cases {passed}/{len(results)} pass") - print("========================================") - green = all(s == "pass" for _, s, _ in results) - print( - "\nGREEN: per-seed isolation, coherent stealth, no failures.\n" - if green - else "\nNOT GREEN: see failures above.\n" - ) - return 0 if green else 1 - - -if __name__ == "__main__": - raise SystemExit(asyncio.run(main())) diff --git a/test/smoke/main.go b/test/smoke/main.go new file mode 100644 index 0000000..4d6c67c --- /dev/null +++ b/test/smoke/main.go @@ -0,0 +1,443 @@ +// Command smoke is the cuttle smoke harness - neutral and self-contained. +// +// It drives a running cuttle over CDP and introspects each seed's browser +// directly - no third-party sites, no network targets, no local server. It +// checks: +// +// 1. per-seed fingerprint isolation - each fingerprint seed gets its own +// coherent identity, so an in-page canvas readback differs across seeds. +// 2. stealth coherence - navigator.webdriver is falsy, the UA/platform agree +// (a Windows UA must not pair with a non-Windows platform), and the WebGL +// UNMASKED_RENDERER reads as a real GPU via ANGLE (not SwiftShader/llvmpipe/ +// Mesa software rendering, the classic automation tell the fork masks). +// 3. connection stability under cold-cycle load - fresh seeds are launched in a +// loop; every cycle must connect and probe without error. +// +// Run: go run ./test/smoke (from the repo root) +// Green = distinct per-seed canvas, coherent stealth signals, no failures. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/coder/websocket" +) + +const ( + defaultCuttleURL = "http://127.0.0.1:9222" + defaultColdCycles = 3 + connectTimeout = 90 * time.Second +) + +const ( + statusPass = "pass" + statusFail = "fail" + nameCanvasIsolation = "canvas-isolation" +) + +// One self-contained expression: build a canvas (farbling is fingerprint-seeded, +// so it differs per seed) and read the stealth signals, returned as JSON. +const probeJS = ` +(() => { + let canvas = "missing"; + try { + const c = document.createElement("canvas"); + c.width = 200; c.height = 40; + const ctx = c.getContext("2d"); + ctx.textBaseline = "top"; + ctx.font = "16px Arial"; + ctx.fillStyle = "#f60"; ctx.fillRect(0, 0, 200, 40); + ctx.fillStyle = "#069"; ctx.fillText("cuttle-smoke", 2, 2); + canvas = c.toDataURL(); + } catch (e) { canvas = "canvas-error:" + e.message; } + let webglVendor = "missing", webglRenderer = "missing"; + try { + const gc = document.createElement("canvas"); + const gl = gc.getContext("webgl") || gc.getContext("experimental-webgl"); + const dbg = gl.getExtension("WEBGL_debug_renderer_info"); + webglVendor = gl.getParameter(dbg.UNMASKED_VENDOR_WEBGL); + webglRenderer = gl.getParameter(dbg.UNMASKED_RENDERER_WEBGL); + } catch (e) { webglRenderer = "webgl-error:" + e.message; } + return JSON.stringify({ + webdriver: navigator.webdriver, + ua: navigator.userAgent, + platform: navigator.platform, + hardwareConcurrency: navigator.hardwareConcurrency, + canvas, + webglVendor, + webglRenderer, + }); +})() +` + +// A raw software-renderer string in the WebGL UNMASKED_RENDERER is the classic +// automation tell the fork exists to hide: it spoofs a real desktop GPU (via +// ANGLE) on top of whatever renders underneath, so any of these substrings means +// the spoof is not engaging - a real stealth regression, not container noise. +var softwareGLMarkers = []string{"swiftshader", "llvmpipe", "mesa", "software"} + +var ( + errNoWSURL = errors.New("multiplexer did not return a webSocketDebuggerUrl") + errCDPCommand = errors.New("cdp command failed") + errEmptyResult = errors.New("cdp command returned no result") +) + +type checkResult struct { + name string + status string + detail string +} + +type probeInfo struct { + Webdriver any `json:"webdriver"` + UA string `json:"ua"` + Platform string `json:"platform"` + HardwareConcurrency int `json:"hardwareConcurrency"` + Canvas string `json:"canvas"` + WebglVendor string `json:"webglVendor"` + WebglRenderer string `json:"webglRenderer"` +} + +func main() { + os.Exit(run(context.Background())) +} + +func run(ctx context.Context) int { + cuttleURL := getenv("CUTTLE_URL", defaultCuttleURL) + coldCycles := getenvInt("COLD_CYCLES", defaultColdCycles) + runID := strconv.FormatInt(time.Now().Unix(), 16) + + fmt.Println("cuttle smoke harness") + fmt.Printf(" CUTTLE_URL = %s\n", cuttleURL) + fmt.Printf(" cold cycles = %d\n", coldCycles) + + var results []checkResult + var canvases []string + + fmt.Println("\n== stealth coherence + connection stability (cold cycles) ==") + for i := 1; i <= coldCycles; i++ { + seed := fmt.Sprintf("smoke-%s-%d", runID, i) + res, canvas := coldCycle(ctx, cuttleURL, seed, i) + results = append(results, res) + if canvas != "" { + canvases = append(canvases, canvas) + } + } + + fmt.Println("\n== per-seed fingerprint isolation ==") + results = append(results, canvasIsolation(canvases)) + + passed := 0 + for _, r := range results { + if r.status == statusPass { + passed++ + } + fmt.Printf(" [%s] %s - %s\n", strings.ToUpper(r.status), r.name, r.detail) + } + + fmt.Println("\n================ SUMMARY ================") + fmt.Printf(" cases %d/%d pass\n", passed, len(results)) + fmt.Println("========================================") + + if passed == len(results) { + fmt.Println("\nGREEN: per-seed isolation, coherent stealth, no failures.") + return 0 + } + fmt.Println("\nNOT GREEN: see failures above.") + return 1 +} + +// coldCycle launches a fresh seed, probes it, and grades the stealth signals. A +// connection or probe failure IS the stability signal, so it is caught and +// recorded as a fail rather than aborting the run. The seed's canvas is returned +// (empty on failure) for the cross-seed isolation check. +func coldCycle(ctx context.Context, cuttleURL, seed string, cycle int) (checkResult, string) { + name := fmt.Sprintf("cold-cycle-%d", cycle) + + cycleCtx, cancel := context.WithTimeout(ctx, connectTimeout+30*time.Second) + defer cancel() + + info, err := probeSeed(cycleCtx, cuttleURL, seed) + if err != nil { + return checkResult{name, statusFail, fmt.Sprintf("probe failed: %v", err)}, "" + } + + var problems []string + if isTruthy(info.Webdriver) { + problems = append(problems, fmt.Sprintf("webdriver=%v", info.Webdriver)) + } + uaWindows := strings.Contains(strings.ToLower(info.UA), "windows") + platformWin := strings.HasPrefix(strings.ToLower(info.Platform), "win") + if uaWindows != platformWin { + problems = append(problems, fmt.Sprintf( + "incoherent ua/platform (ua=%q platform=%q)", truncate(info.UA, 40), info.Platform, + )) + } + if !strings.HasPrefix(info.Canvas, "data:image") { + problems = append(problems, "canvas="+truncate(info.Canvas, 24)) + } + + // The load-bearing one: the WebGL renderer must read as a real GPU via ANGLE, + // not a software renderer. + renderer := info.WebglRenderer + lower := strings.ToLower(renderer) + switch { + case renderer == "" || renderer == "missing": + problems = append(problems, "webgl-renderer-missing") + case containsAny(lower, softwareGLMarkers): + problems = append(problems, fmt.Sprintf("software webgl renderer=%q", truncate(renderer, 60))) + case !strings.Contains(lower, "angle"): + problems = append(problems, fmt.Sprintf("webgl renderer not via ANGLE=%q", truncate(renderer, 60))) + } + + if len(problems) > 0 { + return checkResult{name, statusFail, strings.Join(problems, "; ")}, info.Canvas + } + detail := fmt.Sprintf("webdriver=%v platform=%s canvas=ok webgl=%q seed=%s", + info.Webdriver, info.Platform, truncate(renderer, 48), seed) + return checkResult{name, statusPass, detail}, info.Canvas +} + +func canvasIsolation(canvases []string) checkResult { + values := make([]string, 0, len(canvases)) + distinct := map[string]struct{}{} + for _, v := range canvases { + if strings.HasPrefix(v, "data:image") { + values = append(values, v) + distinct[v] = struct{}{} + } + } + if len(values) < 2 { + return checkResult{nameCanvasIsolation, statusFail, fmt.Sprintf("need >=2 canvas readbacks, got %d", len(values))} + } + if len(distinct) < 2 { + return checkResult{nameCanvasIsolation, statusFail, fmt.Sprintf( + "all %d seeds produced an identical canvas (no per-seed farbling)", len(values), + )} + } + return checkResult{nameCanvasIsolation, statusPass, fmt.Sprintf( + "%d distinct canvas fingerprints across %d seeds", len(distinct), len(values), + )} +} + +// probeSeed resolves the seed's browser WebSocket (which launches the seed), +// opens a raw CDP connection, creates a scratch tab, evaluates the probe, and +// returns the parsed signals. +func probeSeed(ctx context.Context, cuttleURL, seed string) (*probeInfo, error) { + wsURL, err := browserWSForSeed(ctx, cuttleURL, seed) + if err != nil { + return nil, err + } + + dialCtx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + conn, resp, err := websocket.Dial(dialCtx, wsURL, nil) + if err != nil { + return nil, fmt.Errorf("dialing CDP websocket: %w", err) + } + if resp != nil && resp.Body != nil { + _ = resp.Body.Close() + } + defer func() { _ = conn.CloseNow() }() + conn.SetReadLimit(-1) + + client := &cdpClient{conn: conn} + + created, err := client.send(ctx, "Target.createTarget", map[string]any{"url": "about:blank"}, "") + if err != nil { + return nil, err + } + var target struct { + TargetID string `json:"targetId"` + } + if err = json.Unmarshal(created, &target); err != nil { + return nil, fmt.Errorf("decoding createTarget: %w", err) + } + + attached, err := client.send(ctx, "Target.attachToTarget", + map[string]any{"targetId": target.TargetID, "flatten": true}, "") + if err != nil { + return nil, err + } + var session struct { + SessionID string `json:"sessionId"` + } + if err = json.Unmarshal(attached, &session); err != nil { + return nil, fmt.Errorf("decoding attachToTarget: %w", err) + } + + evaluated, err := client.send(ctx, "Runtime.evaluate", map[string]any{ + "expression": probeJS, "returnByValue": true, "awaitPromise": true, + }, session.SessionID) + if err != nil { + return nil, err + } + + if _, err = client.send(ctx, "Target.closeTarget", map[string]any{"targetId": target.TargetID}, ""); err != nil { + return nil, err + } + + var eval struct { + Result struct { + Value string `json:"value"` + } `json:"result"` + } + if err = json.Unmarshal(evaluated, &eval); err != nil { + return nil, fmt.Errorf("decoding evaluate result: %w", err) + } + var info probeInfo + if err = json.Unmarshal([]byte(eval.Result.Value), &info); err != nil { + return nil, fmt.Errorf("decoding probe payload: %w", err) + } + return &info, nil +} + +// cdpClient is a minimal CDP client: send a command, return its result (matching +// on the request id and skipping unrelated events). +type cdpClient struct { + conn *websocket.Conn + id int +} + +func (c *cdpClient) send(ctx context.Context, method string, params map[string]any, sessionID string) (json.RawMessage, error) { + c.id++ + mid := c.id + if params == nil { + params = map[string]any{} + } + req := struct { + ID int `json:"id"` + Method string `json:"method"` + Params map[string]any `json:"params"` + SessionID string `json:"sessionId,omitempty"` + }{ID: mid, Method: method, Params: params, SessionID: sessionID} + + payload, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("encoding %s: %w", method, err) + } + if err := c.conn.Write(ctx, websocket.MessageText, payload); err != nil { + return nil, fmt.Errorf("sending %s: %w", method, err) + } + + for { + _, data, err := c.conn.Read(ctx) + if err != nil { + return nil, fmt.Errorf("reading %s response: %w", method, err) + } + var resp struct { + ID int `json:"id"` + Error json.RawMessage `json:"error"` + Result json.RawMessage `json:"result"` + } + if err := json.Unmarshal(data, &resp); err != nil { + return nil, fmt.Errorf("decoding %s frame: %w", method, err) + } + if resp.ID != mid { + continue + } + if len(resp.Error) > 0 { + return nil, fmt.Errorf("%w: %s: %s", errCDPCommand, method, string(resp.Error)) + } + if len(resp.Result) == 0 { + return nil, fmt.Errorf("%w: %s", errEmptyResult, method) + } + return resp.Result, nil + } +} + +// browserWSForSeed asks cuttle for the seed's browser CDP WebSocket, which also +// launches the seed. The multiplexer rewrites webSocketDebuggerUrl to its own +// host, so the returned URL is correct behind a port-forward. +func browserWSForSeed(ctx context.Context, cuttleURL, seed string) (string, error) { + base, err := url.Parse(cuttleURL) + if err != nil { + return "", fmt.Errorf("parsing CUTTLE_URL %q: %w", cuttleURL, err) + } + base.Path = "/json/version" + base.RawQuery = url.Values{"fingerprint": {seed}}.Encode() + + reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, base.String(), nil) + if err != nil { + return "", fmt.Errorf("building /json/version request: %w", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("reaching cuttle: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", fmt.Errorf("reading /json/version: %w", err) + } + var v struct { + WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` + } + if err := json.Unmarshal(body, &v); err != nil { + return "", fmt.Errorf("decoding /json/version: %w", err) + } + if v.WebSocketDebuggerURL == "" { + return "", errNoWSURL + } + return v.WebSocketDebuggerURL, nil +} + +func getenv(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func getenvInt(key string, fallback int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return fallback +} + +func isTruthy(v any) bool { + switch t := v.(type) { + case nil: + return false + case bool: + return t + case string: + return t != "" + case float64: + return t != 0 + default: + return true + } +} + +func containsAny(s string, subs []string) bool { + for _, sub := range subs { + if strings.Contains(s, sub) { + return true + } + } + return false +} + +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) +} diff --git a/uv.lock b/uv.lock deleted file mode 100644 index 04b31ab..0000000 --- a/uv.lock +++ /dev/null @@ -1,469 +0,0 @@ -version = 1 -revision = 3 -requires-python = "==3.12.*" - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohappyeyeballs" }, - { name = "aiosignal" }, - { name = "attrs" }, - { name = "frozenlist" }, - { name = "multidict" }, - { name = "propcache" }, - { name = "typing-extensions" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, - { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, - { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, - { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, - { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, - { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, - { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, - { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, - { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, - { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, - { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, - { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, - { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, -] - -[[package]] -name = "aiosignal" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "frozenlist" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, -] - -[[package]] -name = "anyio" -version = "4.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, -] - -[[package]] -name = "attrs" -version = "26.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, -] - -[[package]] -name = "certifi" -version = "2026.6.17" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, - { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, - { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, - { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, - { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, - { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, - { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, - { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, - { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, - { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, - { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, - { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, - { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, - { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, -] - -[[package]] -name = "cuttle-browser" -version = "0.3.0" -source = { editable = "." } -dependencies = [ - { name = "aiohttp" }, - { name = "websockets" }, -] - -[package.dev-dependencies] -build = [ - { name = "fonttools" }, -] -dev = [ - { name = "ruff" }, - { name = "ty" }, -] -server = [ - { name = "geoip2" }, - { name = "httpx" }, - { name = "socksio" }, -] - -[package.metadata] -requires-dist = [ - { name = "aiohttp", specifier = ">=3.9" }, - { name = "websockets", specifier = ">=12.0" }, -] - -[package.metadata.requires-dev] -build = [{ name = "fonttools", specifier = ">=4.0" }] -dev = [ - { name = "ruff", specifier = ">=0.15.0" }, - { name = "ty", specifier = ">=0.0.30" }, -] -server = [ - { name = "geoip2", specifier = ">=4.0" }, - { name = "httpx", specifier = ">=0.27" }, - { name = "socksio", specifier = ">=1.0" }, -] - -[[package]] -name = "fonttools" -version = "4.63.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/84/69/c97f2c18e0db87d2c7b15da1974dace76ae938f1cfa22e2727a648b7ed43/fonttools-4.63.0.tar.gz", hash = "sha256:caeb583deeb5168e694b65cda8b4ee62abedfa66cf88488734466f2366b9c4e0", size = 3597189, upload-time = "2026-05-14T12:04:30.958Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/ef/b3c6b9b5be2f82416d73fe2ed2e96e2793cd80e7510bd6a17ca79cdd88ec/fonttools-4.63.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:37dd23e621e3b0aef1baa70a303b80aaf38449632cfc8fd2a55fb285bbccfc02", size = 2881131, upload-time = "2026-05-14T12:03:13.386Z" }, - { url = "https://files.pythonhosted.org/packages/44/a0/c815bea63117fa63e4e1c01f8a1110d2112fa003f838e6467094ec2432ce/fonttools-4.63.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a9faff9e0c1f76f9fd55899d2ce785832efebab37eb8ae13995853aef178bef0", size = 2426704, upload-time = "2026-05-14T12:03:15.801Z" }, - { url = "https://files.pythonhosted.org/packages/44/04/0b91d8e916e92ad1fac9e4624760baf0fd5ff2ead614c2f68fb21373f03f/fonttools-4.63.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef3048ef05dbb552b89817713d9cac912e00d0fde4a3105c00d29e52e10c89af", size = 5044298, upload-time = "2026-05-14T12:03:18.085Z" }, - { url = "https://files.pythonhosted.org/packages/77/c7/2342da9830e3e9d4870305ca5d2091d2a83284f2953079b7bdd3b5e029d8/fonttools-4.63.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58dc6bb86a78d782f00f9190ca02c119cf5bbe2807536e361e18d42019f877d8", size = 4999800, upload-time = "2026-05-14T12:03:20.161Z" }, - { url = "https://files.pythonhosted.org/packages/e6/6d/67fe16c48d7ce050979b33f47e0d28a318f02da030602e944c34f7a16ef3/fonttools-4.63.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee08ebfa58f6e1aeff5697ab9582105bb620008c1caafb681e4c557e7483027b", size = 4982666, upload-time = "2026-05-14T12:03:22.87Z" }, - { url = "https://files.pythonhosted.org/packages/f2/00/3bbab338c07c71fa56269953845e92c951a61457bbbb0f1022551ea266d9/fonttools-4.63.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27fdc65af8da6f88b9c6121c47a464cbe359fcfff7ff6fc2d37a1f395d755b78", size = 5133598, upload-time = "2026-05-14T12:03:25.168Z" }, - { url = "https://files.pythonhosted.org/packages/62/f2/aa27c7f98db5b064883dadcc5283947e81e034de42e22a33675878d98b54/fonttools-4.63.0-cp312-cp312-win32.whl", hash = "sha256:af2fd1664d00a397d75f806985ddb36282091c2131a73a6485c23b4a34722263", size = 2292575, upload-time = "2026-05-14T12:03:27.496Z" }, - { url = "https://files.pythonhosted.org/packages/87/36/cccb9bc2a6ab63d1b2980374f0dca72ce95ae267c9b4cfe77455bb70d0d4/fonttools-4.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:59ac449f8cca9b4ffa08d2e7bbadad87ce710d69d1eda5c3c1ce579baa987272", size = 2343211, upload-time = "2026-05-14T12:03:30.057Z" }, - { url = "https://files.pythonhosted.org/packages/2c/47/c99d5268f354002ce80f8d029cd9d7d872969da1de8b93d32de4dc56d6f4/fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d", size = 1164562, upload-time = "2026-05-14T12:04:29.092Z" }, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, - { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, - { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, - { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, - { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, - { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, - { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, - { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, - { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, - { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, - { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, - { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, - { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, - { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, -] - -[[package]] -name = "geoip2" -version = "5.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "maxminddb" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/70/3d9e87289f79713aaf0fea9df4aa8e68776640fe59beb6299bb214610cfd/geoip2-5.2.0.tar.gz", hash = "sha256:6c9ded1953f8eb16043ed0a8ea20e6e9524ea7b65eb745724e12490aca44ef00", size = 176498, upload-time = "2025-11-20T18:21:08.874Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/d2/d55df737199a52b9d06e742ed2a608c525f0677e40375951372e65714fbd/geoip2-5.2.0-py3-none-any.whl", hash = "sha256:3d1546fd4eb7cad20445d027d2d9e81d3a71c074e019383f30db5d45e2c23320", size = 28991, upload-time = "2025-11-20T18:21:07.178Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "idna" -version = "3.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, -] - -[[package]] -name = "maxminddb" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/83/bcd7f2e7dfcf601258a4eab92155816218e8f8adf6608d5f7d39da7ba863/maxminddb-3.1.1.tar.gz", hash = "sha256:b19a938c481518f19a2c534ffdcb3bc59582f0fbbdcf9f81ac9adf912a0af686", size = 212410, upload-time = "2026-03-05T18:14:19.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/32/331c6c0ce56aacee7f71b9b7ef2438ae74b2d788cd56d4a58cd3be3e6bf8/maxminddb-3.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ae6489a1b7fa4ab9b6ac5979d1eec1eed7cb7ef2f73777ddbb8fb8b9bec094e3", size = 54257, upload-time = "2026-03-05T18:13:10.656Z" }, - { url = "https://files.pythonhosted.org/packages/69/6d/4fc324d46b764e870847fc50d7e3b0154dbf165d04d121653066069e3d2c/maxminddb-3.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8f41a51bce83b5bbe4dc31b080787b7d4d83d8efa98778eb6f81df3ad9e98734", size = 36314, upload-time = "2026-03-05T18:13:11.75Z" }, - { url = "https://files.pythonhosted.org/packages/22/d5/436054930ccd384f2f6e17aa7689207e9334abbaed63bb573d76dbe0ee8f/maxminddb-3.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9cd4c05d08c22796e83aa54c70feb64121b3eae7257af35fbaced9f5d8d2081", size = 36105, upload-time = "2026-03-05T18:13:13.019Z" }, - { url = "https://files.pythonhosted.org/packages/40/a2/bbded436f06c38716163f87d7d92a62f7d305d2f9f7e2e4155f8749a9f1e/maxminddb-3.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c4a402180154393c9c2502c7704b10a32a065661cd84196bbe7ac56869c6a82", size = 103427, upload-time = "2026-03-05T18:13:13.981Z" }, - { url = "https://files.pythonhosted.org/packages/c8/a4/8464f1d29007736cfe1e0aa2dd1f3a36f5d7020abb9f14269b614a3f3ae1/maxminddb-3.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14d8b40d8e9b288cee18b8d80a7ba2a28211ce07b9c0e6ce721c5e685e3bf23c", size = 101252, upload-time = "2026-03-05T18:13:15.064Z" }, - { url = "https://files.pythonhosted.org/packages/1f/9d/d3c95b64c05e90091ddef22a7a1bcdabe998f36b4a9f4b814382f712d83f/maxminddb-3.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4eb2644548114b22d5808972d6b2b77d4c62084966b9a6be3853cd173ff745d5", size = 100587, upload-time = "2026-03-05T18:13:16.077Z" }, - { url = "https://files.pythonhosted.org/packages/e0/0a/7e078abe41896d771e2917bc390fcd13186cd9b1b4ef8b451019f1fc342a/maxminddb-3.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a529873e376ada254c68a54d3ad13c8265eeedc5c56bdfdf25f1044b7f4177b", size = 99126, upload-time = "2026-03-05T18:13:18.001Z" }, - { url = "https://files.pythonhosted.org/packages/bb/89/f07411f340b70374d1b76b710b059cfc9874e22f596df3f20d26ebbece5b/maxminddb-3.1.1-cp312-cp312-win32.whl", hash = "sha256:9d98641b111eecc047b560d927379dd044bb36c0e399ee794e865b75ee8ef27a", size = 35632, upload-time = "2026-03-05T18:13:20.322Z" }, - { url = "https://files.pythonhosted.org/packages/a3/cc/1e42136d8416bbcaa81cdfdb26ec75263f106c0c34bf357ca98eebc394d0/maxminddb-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a1ad6d3790ca4b2936f3e4ea971ef8383480fa069ed8ea4e5e6345f049d0e9f7", size = 37332, upload-time = "2026-03-05T18:13:21.363Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ed/f3a6b030eef252d4eaa811c87ad3a1d44376de581b11be8246fda1fc3716/maxminddb-3.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:c295f90ce99ce434d6a8477bd94ba4869ca04fe617ac99cea7548f0f6a3e4cd8", size = 34231, upload-time = "2026-03-05T18:13:22.647Z" }, -] - -[[package]] -name = "multidict" -version = "6.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, - { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, - { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, - { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, - { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, - { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, - { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, - { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, - { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, - { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, - { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, - { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, - { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, - { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, - { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, - { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, - { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, - { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, -] - -[[package]] -name = "propcache" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, - { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, - { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, - { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, - { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, - { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, - { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, - { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, - { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, - { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, - { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, - { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, - { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, - { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, - { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, -] - -[[package]] -name = "requests" -version = "2.34.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, -] - -[[package]] -name = "ruff" -version = "0.15.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566", size = 4755489, upload-time = "2026-06-25T17:20:37.578Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078", size = 10839665, upload-time = "2026-06-25T17:19:44.702Z" }, - { url = "https://files.pythonhosted.org/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b", size = 11208649, upload-time = "2026-06-25T17:19:48.787Z" }, - { url = "https://files.pythonhosted.org/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632", size = 10622638, upload-time = "2026-06-25T17:19:51.354Z" }, - { url = "https://files.pythonhosted.org/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd", size = 10984227, upload-time = "2026-06-25T17:19:54.044Z" }, - { url = "https://files.pythonhosted.org/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b", size = 10622882, upload-time = "2026-06-25T17:19:57.037Z" }, - { url = "https://files.pythonhosted.org/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267", size = 11474808, upload-time = "2026-06-25T17:20:00.357Z" }, - { url = "https://files.pythonhosted.org/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c", size = 12293094, upload-time = "2026-06-25T17:20:03.446Z" }, - { url = "https://files.pythonhosted.org/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae", size = 11526176, upload-time = "2026-06-25T17:20:06.301Z" }, - { url = "https://files.pythonhosted.org/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b", size = 11520767, upload-time = "2026-06-25T17:20:09.191Z" }, - { url = "https://files.pythonhosted.org/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487", size = 11500132, upload-time = "2026-06-25T17:20:13.602Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3", size = 10943828, upload-time = "2026-06-25T17:20:16.635Z" }, - { url = "https://files.pythonhosted.org/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053", size = 10645418, upload-time = "2026-06-25T17:20:19.4Z" }, - { url = "https://files.pythonhosted.org/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4", size = 11211770, upload-time = "2026-06-25T17:20:22.033Z" }, - { url = "https://files.pythonhosted.org/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460", size = 11618698, upload-time = "2026-06-25T17:20:25.259Z" }, - { url = "https://files.pythonhosted.org/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21", size = 10857322, upload-time = "2026-06-25T17:20:28.612Z" }, - { url = "https://files.pythonhosted.org/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415", size = 11993274, upload-time = "2026-06-25T17:20:31.871Z" }, - { url = "https://files.pythonhosted.org/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca", size = 11343498, upload-time = "2026-06-25T17:20:35.03Z" }, -] - -[[package]] -name = "socksio" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/5c/48a7d9495be3d1c651198fd99dbb6ce190e2274d0f28b9051307bdec6b85/socksio-1.0.0.tar.gz", hash = "sha256:f88beb3da5b5c38b9890469de67d0cb0f9d494b78b106ca1845f96c10b91c4ac", size = 19055, upload-time = "2020-04-17T15:50:34.664Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/c3/6eeb6034408dac0fa653d126c9204ade96b819c936e136c5e8a6897eee9c/socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3", size = 12763, upload-time = "2020-04-17T15:50:31.878Z" }, -] - -[[package]] -name = "ty" -version = "0.0.57" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/16/d01c968d405acae51c07872e80f30f3a586235bdf52c9847ca0917a230a3/ty-0.0.57.tar.gz", hash = "sha256:bc058f564868690283a0420d09c269ec8be21e8e43b4b49ee975a17623092e44", size = 6100787, upload-time = "2026-07-08T11:33:59.904Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/9c/a7948b05f2a3f43d511f88ef5c4f56d7edb8acc8caa5f56d7c5831f52c84/ty-0.0.57-py3-none-linux_armv6l.whl", hash = "sha256:cb6d3371dd8c78950b75bee31a36b94564a54a1f0eecafbbf05715ac1a5287d6", size = 11760058, upload-time = "2026-07-08T11:33:18.671Z" }, - { url = "https://files.pythonhosted.org/packages/6e/92/3776380decba3965bcaa1ea2b56f5b133aa3ef549bfbe4b79eb122dcc15d/ty-0.0.57-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e58b90491a48ec757bd50f512f3eb92c1c64b7d46a4db83677e9b60222e1d2bb", size = 11492105, upload-time = "2026-07-08T11:33:21.267Z" }, - { url = "https://files.pythonhosted.org/packages/13/be/912f8422d06fd1e29805a7c781e3ce4c821917e0e3d00f0cf176c0529469/ty-0.0.57-py3-none-macosx_11_0_arm64.whl", hash = "sha256:dbb8207f75122c658ca21bf405cea8202e490240440461ae3e0c5a6f67ae668f", size = 11078675, upload-time = "2026-07-08T11:33:23.804Z" }, - { url = "https://files.pythonhosted.org/packages/06/86/6ed526df554b491ce3d07bbec04f7a10304243bc994ab989352c85134b1e/ty-0.0.57-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:651243f391809de80b01be1729c5d0cecc7b952d7d22cfbf56f0b4f069703195", size = 11614379, upload-time = "2026-07-08T11:33:26.041Z" }, - { url = "https://files.pythonhosted.org/packages/0d/95/78af20f309abce31fa616e0142f85756e665a9965669b823181ff695ffec/ty-0.0.57-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:28c5392bbd7c4d6a4c1b1646a709231db675dc094f49bf71b7de83757125e93b", size = 11563854, upload-time = "2026-07-08T11:33:28.144Z" }, - { url = "https://files.pythonhosted.org/packages/88/dc/bf9231d5563b9e61a1eec9782184a4055afaa87259644cd9ed1376a7dc5c/ty-0.0.57-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d981535094ab492aaef6f71d9348bcf90e42e6de4cb50de2dd7fbabd8378360", size = 12229897, upload-time = "2026-07-08T11:33:30.262Z" }, - { url = "https://files.pythonhosted.org/packages/c0/dd/012008190a997097ebe500b9704baaad9135bfa033b9530a9b8f69f1d11f/ty-0.0.57-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:719b41fd6df61352866cad952d7bfb4873c893a70e806d37d0f1d30ad4f26cd5", size = 12816002, upload-time = "2026-07-08T11:33:32.747Z" }, - { url = "https://files.pythonhosted.org/packages/f4/2f/0e42c361e38e04747ed2b3d3299512edd4ea6060d8e3124e3c6f2d2465a1/ty-0.0.57-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a1641a609b025e13f44115c7071b39b02675044a78fbfedef239a5f7da50b393", size = 12323420, upload-time = "2026-07-08T11:33:35.426Z" }, - { url = "https://files.pythonhosted.org/packages/c6/01/18f1e03108d1ae8e515c92fb304994a66eaafd47037f928fd42fd3a1e29d/ty-0.0.57-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a33f96a9dd6919fe8b1d3512cd99f16be4a51388f265de135fea2c2fbf0b4317", size = 12119481, upload-time = "2026-07-08T11:33:37.808Z" }, - { url = "https://files.pythonhosted.org/packages/6a/48/570b73fb3e32554ff03aa9f6650b33a504cdfa027a8b50a5ab087e56b20d/ty-0.0.57-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:f0a1014a922b2b7f79a46e5cdbaa6feb7403a444631292b8666382a98a04de20", size = 12427299, upload-time = "2026-07-08T11:33:39.964Z" }, - { url = "https://files.pythonhosted.org/packages/b7/37/76b27f6a92e12525d09cd65d3c3b5aea419cf4782b13320db95ac3d310f0/ty-0.0.57-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:d73aaed84023bf682819f871377b255fae9098898c50475426ac9bdfcd986872", size = 11562292, upload-time = "2026-07-08T11:33:42.392Z" }, - { url = "https://files.pythonhosted.org/packages/84/8b/d0c398147b00f336595e1a00a5895b3924251205c11b8d73cb0668281f1c/ty-0.0.57-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f3e2104704063c00ab8a757af3e95378b32624a3e8d8149aad5251877bf82959", size = 11565833, upload-time = "2026-07-08T11:33:44.778Z" }, - { url = "https://files.pythonhosted.org/packages/3e/74/dab9745e977ef38751ec710f74e40d21fddbd04a80338dd5597c98899eed/ty-0.0.57-py3-none-musllinux_1_2_i686.whl", hash = "sha256:07ad760763646d8f1567ea5d899e6d217212acd8539b7afed5a370d93693553b", size = 11864967, upload-time = "2026-07-08T11:33:47.36Z" }, - { url = "https://files.pythonhosted.org/packages/a2/57/350812143b49dac7aa655f40a1f45d4821ca9b56b75d9b68d5e603269044/ty-0.0.57-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4a6e1f0fee7df3e65fb0b9cbe9f99a4be39198558ed9aacc31a98d210ad7bcc", size = 12239054, upload-time = "2026-07-08T11:33:49.586Z" }, - { url = "https://files.pythonhosted.org/packages/a4/d0/b3e3d9c6cce3debda6247aa435e81d18ec62fabfec13f35f6c6957066f47/ty-0.0.57-py3-none-win32.whl", hash = "sha256:f8d488c0535a8f0386dbe2c9bcb31d467ae0c68d0c9945113018937828ebaabb", size = 11225353, upload-time = "2026-07-08T11:33:52.569Z" }, - { url = "https://files.pythonhosted.org/packages/77/db/6ce240ee31413f9dc7467e31377553e92e2664252ee7d33aa9290a7a94bb/ty-0.0.57-py3-none-win_amd64.whl", hash = "sha256:de9529a7dcc3e529b08c14634dc4e7066ebea5cd55006afc02bde8033efe7c91", size = 12272419, upload-time = "2026-07-08T11:33:54.889Z" }, - { url = "https://files.pythonhosted.org/packages/44/de/48662fa2c42289f309eeb8b5539cbe840e10002b65ac0700cb7889e92532/ty-0.0.57-py3-none-win_arm64.whl", hash = "sha256:7f3352777ce40c4906145f3c3e10b71716d8d35c0c9e3a0070d84f61dda0755f", size = 11686309, upload-time = "2026-07-08T11:33:57.246Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, -] - -[[package]] -name = "websockets" -version = "16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00", size = 177365, upload-time = "2026-01-10T09:22:46.787Z" }, - { url = "https://files.pythonhosted.org/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79", size = 175038, upload-time = "2026-01-10T09:22:47.999Z" }, - { url = "https://files.pythonhosted.org/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39", size = 175328, upload-time = "2026-01-10T09:22:49.809Z" }, - { url = "https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c", size = 184915, upload-time = "2026-01-10T09:22:51.071Z" }, - { url = "https://files.pythonhosted.org/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f", size = 186152, upload-time = "2026-01-10T09:22:52.224Z" }, - { url = "https://files.pythonhosted.org/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1", size = 185583, upload-time = "2026-01-10T09:22:53.443Z" }, - { url = "https://files.pythonhosted.org/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2", size = 184880, upload-time = "2026-01-10T09:22:55.033Z" }, - { url = "https://files.pythonhosted.org/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89", size = 178261, upload-time = "2026-01-10T09:22:56.251Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea", size = 178693, upload-time = "2026-01-10T09:22:57.478Z" }, - { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, -] - -[[package]] -name = "yarl" -version = "1.24.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "multidict" }, - { name = "propcache" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/da/866bcb01076ba49d2b42b309867bed3826421f1c479655eb7a607b44f20b/yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8", size = 129957, upload-time = "2026-05-19T21:28:51.695Z" }, - { url = "https://files.pythonhosted.org/packages/bf/1d/fcefb70922ea2268a8971d8e5874d9a8218644200fb8465f1dcad55e6851/yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2", size = 92164, upload-time = "2026-05-19T21:28:53.242Z" }, - { url = "https://files.pythonhosted.org/packages/29/b6/170e2b8d4e3bc30e6bfdcca53556537f5bf595e938632dfcb059311f3ff6/yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d", size = 91688, upload-time = "2026-05-19T21:28:54.865Z" }, - { url = "https://files.pythonhosted.org/packages/fe/a5/c9f655d5553ea0b99fdac9d6a99ad3f9b3e73b8e5758bb46f58c9831f74c/yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035", size = 102902, upload-time = "2026-05-19T21:28:56.963Z" }, - { url = "https://files.pythonhosted.org/packages/5d/bc/6b9664d815d79af4ee553337f9d606c56bbf269186ada9172de45f1b5f60/yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576", size = 97931, upload-time = "2026-05-19T21:28:58.56Z" }, - { url = "https://files.pythonhosted.org/packages/98/ec/32ba48acae30fecd60928f5791188b80a9d6ee3840507ffda29fecd37b71/yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8", size = 111030, upload-time = "2026-05-19T21:29:00.148Z" }, - { url = "https://files.pythonhosted.org/packages/82/5a/6f4cd081e5f4934d2ae3a8ef4abe3afacc010d26f0035ee91b35cd7d7c37/yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7", size = 110392, upload-time = "2026-05-19T21:29:02.155Z" }, - { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612, upload-time = "2026-05-19T21:29:04.247Z" }, - { url = "https://files.pythonhosted.org/packages/7c/80/264ab684f181e1a876389374519ff05d10248725535ae2ac4e8ac4e563d6/yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d", size = 104487, upload-time = "2026-05-19T21:29:06.491Z" }, - { url = "https://files.pythonhosted.org/packages/41/07/efabe5df87e96d7ad5959760b888344be48cd6884db127b407c6b5503adc/yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db", size = 102333, upload-time = "2026-05-19T21:29:08.267Z" }, - { url = "https://files.pythonhosted.org/packages/44/0c/bcf7c42603e1009295f586d8890f2ba032c8b53310e815adf0a202c73d9f/yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712", size = 99025, upload-time = "2026-05-19T21:29:10.682Z" }, - { url = "https://files.pythonhosted.org/packages/4f/82/84482ab1a57a0f21a08afe6a7004c61d741f8f2ecc3b05c321577c612164/yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996", size = 110507, upload-time = "2026-05-19T21:29:12.954Z" }, - { url = "https://files.pythonhosted.org/packages/c4/8d/a546ba1dfe1b0f290e05fef145cd07614c0f15df1a707195e512d1e39d1d/yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b", size = 103719, upload-time = "2026-05-19T21:29:14.893Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b6/267f2a09213138473adfce6b8a6e17791d7fee70bd4d9003218e4dec58b0/yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c", size = 110438, upload-time = "2026-05-19T21:29:16.485Z" }, - { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719, upload-time = "2026-05-19T21:29:18.367Z" }, - { url = "https://files.pythonhosted.org/packages/a7/25/722e3b93bd687009afb2d59a35e13d30ddd8f80571445bb0c4e4ce26ec66/yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad", size = 92901, upload-time = "2026-05-19T21:29:20.014Z" }, - { url = "https://files.pythonhosted.org/packages/39/47/4486ccfb674c04854a1ef8aa77868b6a6f765feaf69633409d7ca4f02cb8/yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30", size = 87229, upload-time = "2026-05-19T21:29:22.1Z" }, - { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, -] diff --git a/vendor/cloakbrowser/__init__.py b/vendor/cloakbrowser/__init__.py deleted file mode 100644 index 272e793..0000000 --- a/vendor/cloakbrowser/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""cuttle: minimal MIT-vendored CDP argument-builders for the stealth-browser farm.""" diff --git a/vendor/cloakbrowser/_version.py b/vendor/cloakbrowser/_version.py deleted file mode 100644 index 3dc1f76..0000000 --- a/vendor/cloakbrowser/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.1.0" diff --git a/vendor/cloakbrowser/browser.py b/vendor/cloakbrowser/browser.py deleted file mode 100644 index 9daf731..0000000 --- a/vendor/cloakbrowser/browser.py +++ /dev/null @@ -1,310 +0,0 @@ -"""CDP launch-argument builders for cuttle. - -Vendored MIT subset of cloakbrowser: only the stealth-argument builders the CDP -multiplexer uses (build_args, maybe_resolve_geoip, _resolve_webrtc_args, -_normalize_socks_string_url) plus their private proxy helpers. The full browser -launch API and its paid-tier machinery are intentionally not vendored. -""" - -from __future__ import annotations - -import logging -import os -from typing import TypedDict -from urllib.parse import quote, unquote, urlparse, urlunparse - -from .config import get_default_stealth_args - -logger = logging.getLogger("cloakbrowser") - - -class _ProxySettingsRequired(TypedDict): - server: str - - -class ProxySettings(_ProxySettingsRequired, total=False): - """Playwright-compatible proxy configuration.""" - - bypass: str - username: str - password: str - - -def _ensure_proxy_scheme(proxy_url: str) -> str: - """Prepend http:// to schemeless proxy URLs so parsers can extract hostname.""" - return proxy_url if "://" in proxy_url else f"http://{proxy_url}" - - -def _assemble_proxy_url( - scheme: str, - host: str, - port: int | None, - enc_user: str, - enc_pass: str | None, - path: str = "", - params: str = "", - query: str = "", - fragment: str = "", -) -> str: - """Build a proxy URL from already-percent-encoded credentials and host parts. - - ``enc_pass is None`` means no password (no colon in userinfo). Empty string - means present-but-empty (colon preserved). This mirrors the distinction - urlparse makes between ``user@host`` and ``user:@host``. - """ - if ":" in host: # IPv6 literal โ€” re-add brackets - host = f"[{host}]" - if enc_pass is not None: - userinfo = f"{enc_user}:{enc_pass}@" - elif enc_user: - userinfo = f"{enc_user}@" - else: - userinfo = "" - netloc = f"{userinfo}{host}" - if port is not None: - netloc += f":{port}" - return urlunparse((scheme, netloc, path, params, query, fragment)) - - -def _reconstruct_socks_url(proxy: ProxySettings) -> str: - """Reconstruct a SOCKS5 URL with inline credentials from a Playwright proxy dict.""" - server = proxy.get("server", "") - username = proxy.get("username", "") - password = proxy.get("password", "") - if not username: - return server - parsed = urlparse(server) - enc_user = quote(username, safe="") - # Dict convention: empty/missing password โ†’ no colon. - enc_pass = quote(password, safe="") if password else None - return _assemble_proxy_url( - parsed.scheme, parsed.hostname or "", parsed.port, - enc_user, enc_pass, parsed.path, - ) - - -def _normalize_socks_string_url(url: str) -> str: - """Re-encode credentials in a SOCKS5 URL string so Chromium's parser doesn't - truncate them at special chars like '='. Idempotent: pre-encoded input stays - the same (decoded then re-encoded). - - Emits an INFO log when re-encoding actually changes the URL, so users who - previously hit silent SOCKS5 fallback (#157) can see what the wrapper did. - Silent on already-encoded inputs (no false-positive noise). - - On unparseable input (invalid port, broken IPv6 literal, etc.) logs a - warning and returns the original string โ€” preserves pre-fix pass-through - behavior so Chromium's own error handling kicks in. - """ - try: - parsed = urlparse(url) - # Accessing .port raises ValueError on invalid port strings. - _ = parsed.port - except ValueError as e: - logger.warning("Malformed SOCKS5 proxy URL, passing through unchanged: %s", e) - return url - # Skip only if no credentials at all (username AND password both absent). - # urlparse returns None for absent components, "" for present-but-empty. - if parsed.username is None and parsed.password is None: - return url - raw_user = parsed.username or "" - enc_user = quote(unquote(raw_user), safe="") if raw_user else "" - # Preserve the colon separator when password component is present, even if - # empty, so `user:@host` stays `user:@host`. - if parsed.password is not None: - raw_pass = parsed.password - enc_pass = quote(unquote(raw_pass), safe="") if raw_pass else "" - else: - raw_pass = None - enc_pass = None - normalized = _assemble_proxy_url( - parsed.scheme, parsed.hostname or "", parsed.port, - enc_user, enc_pass, - parsed.path, parsed.params, parsed.query, parsed.fragment, - ) - # Compare credentials, not the full URL: urlparse cosmetically lowercases - # scheme and hostname, so a full-string compare would falsely fire on - # `socks5://USER:pass@HOST.com:1080` even when no encoding work happened. - if enc_user != raw_user or enc_pass != raw_pass: - logger.info( - "Auto URL-encoded SOCKS5 proxy credentials (special characters " - "detected). Pre-encode the URL to suppress this notice." - ) - return normalized - - -def _extract_proxy_url(proxy: str | ProxySettings | None) -> str | None: - """Extract and normalize proxy URL string from proxy param. - - For SOCKS5 dicts with separate username/password fields, reconstructs - the full URL with inline credentials so SOCKS5 auth works. - """ - if proxy is None: - return None - if isinstance(proxy, dict): - server = proxy.get("server", "") - if not server: - return None - if _is_socks_proxy(proxy): - return _reconstruct_socks_url(proxy) - return _ensure_proxy_scheme(server) - return _ensure_proxy_scheme(proxy) - - -def _is_socks_proxy(proxy: str | ProxySettings | None) -> bool: - """Check if the proxy uses SOCKS5 protocol.""" - if proxy is None: - return False - url = proxy.get("server", "") if isinstance(proxy, dict) else proxy - return url.lower().startswith(("socks5://", "socks5h://")) - - -def maybe_resolve_geoip( - geoip: bool, - proxy: str | ProxySettings | None, - timezone: str | None, - locale: str | None, -) -> tuple[str | None, str | None, str | None]: - """Auto-fill timezone/locale from the egress IP when geoip is enabled. - - Returns ``(timezone, locale, exit_ip)``. *exit_ip* is a free bonus - from the geoip lookup (no extra HTTP call) โ€” used for WebRTC spoofing. - - With a proxy the egress IP is the proxy's exit IP; with no proxy it is - the machine's own public IP, so geoip works proxy-free too. - """ - if not geoip: - return timezone, locale, None - - from .geoip import resolve_proxy_exit_ip, resolve_proxy_geo_with_ip - - # None when no proxy โ†’ echo services resolve the machine's own public IP - proxy_url = _extract_proxy_url(proxy) if proxy else None - - # When both tz/locale are explicit, resolve the exit IP for WebRTC โ€” but only - # with a proxy. With no proxy the WebRTC IP would just be the real connection - # IP the site already sees (a no-op), so skip the third-party echo call. - if timezone is not None and locale is not None: - exit_ip = resolve_proxy_exit_ip(proxy_url) if proxy_url else None - return timezone, locale, exit_ip - - geo_tz, geo_locale, exit_ip = resolve_proxy_geo_with_ip(proxy_url) - if timezone is None: - timezone = geo_tz - if locale is None: - locale = geo_locale - return timezone, locale, exit_ip - - -def _resolve_webrtc_args( - args: list[str] | None, - proxy: str | ProxySettings | None, -) -> list[str] | None: - """Replace --fingerprint-webrtc-ip=auto with the resolved proxy exit IP. - - Returns args unchanged if no ``auto`` value is present. - """ - if not args: - return args - idx = None - for i, a in enumerate(args): - if a == "--fingerprint-webrtc-ip=auto": - idx = i - break - if idx is None: - return args - proxy_url = _extract_proxy_url(proxy) - if not proxy_url: - logger.warning("--fingerprint-webrtc-ip=auto requires a proxy; removing flag") - args = list(args) - del args[idx] - return args - try: - from .geoip import resolve_proxy_exit_ip - exit_ip = resolve_proxy_exit_ip(proxy_url) - except Exception: - logger.warning("Failed to resolve proxy exit IP for WebRTC spoofing; removing --fingerprint-webrtc-ip=auto") - args = list(args) - del args[idx] - return args - if exit_ip: - args = list(args) - args[idx] = f"--fingerprint-webrtc-ip={exit_ip}" - else: - logger.warning("Could not resolve proxy exit IP for WebRTC spoofing; removing --fingerprint-webrtc-ip=auto") - args = list(args) - del args[idx] - return args - - -def build_args( - stealth_args: bool, - extra_args: list[str] | None, - timezone: str | None = None, - locale: str | None = None, - headless: bool = True, - extension_paths: list[str] | None = None, - start_maximized: bool = False, -) -> list[str]: - """Combine stealth args with user-provided args and locale flags. - - Deduplicates by flag key (everything before '='). - Priority: stealth defaults < user args < dedicated params (timezone/locale). - """ - seen: dict[str, str] = {} - - if stealth_args: - for arg in get_default_stealth_args(): - seen[arg.split("=", 1)[0]] = arg - - # GPU blocklist bypass: - # - Headed mode (all platforms): Chromium blocks WebGL on software GPUs - # in Docker/Xvfb. Flag lets SwiftShader serve WebGL. See issue #56. - # - Windows (all modes): Chromium's GPU blocklist blocks WebGPU for the - # Microsoft Basic Render Driver. Dawn's adapter_blocklist bypass alone - # isn't enough โ€” need this flag too. Linux doesn't need it. - import platform as _platform - if not headless or _platform.system() == "Windows": - seen["--ignore-gpu-blocklist"] = "--ignore-gpu-blocklist" - - if extra_args: - for arg in extra_args: - key = arg.split("=", 1)[0] - if key in seen: - logger.debug("Arg override: %s -> %s", seen[key], arg) - seen[key] = arg - - # Timezone/locale flags are independent of stealth_args โ€” always inject when set - if timezone: - key = "--fingerprint-timezone" - flag = f"{key}={timezone}" - if key in seen: - logger.debug("Arg override: %s -> %s", seen[key], flag) - seen[key] = flag - if locale: - for key in ("--lang", "--fingerprint-locale"): - flag = f"{key}={locale}" - if key in seen: - logger.debug("Arg override: %s -> %s", seen[key], flag) - seen[key] = flag - - if extension_paths: - abs_paths = [os.path.abspath(p) for p in extension_paths] - ext_val = ",".join(abs_paths) - - seen["--load-extension"] = f"--load-extension={ext_val}" - seen["--disable-extensions-except"] = ( - f"--disable-extensions-except={ext_val}" - ) - - # Open maximized (real Windows Chrome overwhelmingly runs maximized) so the - # window fills the spoofed screen. Skipped if the caller already chose a - # window geometry. Gated to binaries where this stays coherent (see - # binary_supports_maximized_window) โ€” below the gate it would create - # outerWidth < innerWidth. - if start_maximized and not any( - k in seen for k in ("--start-maximized", "--window-size", "--window-position") - ): - seen["--start-maximized"] = "--start-maximized" - - return list(seen.values()) diff --git a/vendor/cloakbrowser/config.py b/vendor/cloakbrowser/config.py deleted file mode 100644 index 3bdd78d..0000000 --- a/vendor/cloakbrowser/config.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Stealth configuration and platform detection for cuttle.""" - -from __future__ import annotations - -import os -import platform -import random -import re -from pathlib import Path - - -IGNORE_DEFAULT_ARGS = ["--enable-automation", "--enable-unsafe-swiftshader"] - - -def get_default_stealth_args() -> list[str]: - """Build stealth args with a random fingerprint seed per launch. - - On macOS, skips platform/GPU spoofing โ€” runs as a native Mac browser. - Spoofing Windows on Mac creates detectable mismatches (fonts, GPU, etc.). - """ - seed = random.randint(10000, 99999) - system = platform.system() - - base = [ - "--no-sandbox", - f"--fingerprint={seed}", - ] - - if system == "Darwin": - # Tell the fingerprint patches we're on macOS so GPU/UA match natively - return base + ["--fingerprint-platform=macos"] - - # Linux/Windows: Windows fingerprint profile. - # Screen and window size come from the real display, not this flag (verified: - # identical across seeds), so the wrapper must not emulate a viewport on top in - # headed mode โ€” that would break outerWidth >= innerWidth coherence. - return base + ["--fingerprint-platform=windows"] - - -DEFAULT_VIEWPORT = {"width": 1920, "height": 947} - - -_VERSION_PIN_RE = re.compile(r"^[0-9]+(?:\.[0-9]+){3,4}$") - - -def normalize_requested_version(version: str | None = None) -> str | None: - """Return an explicit Chromium version pin from arg/env, or None. - - The explicit argument wins over CLOAKBROWSER_VERSION. Only numeric dotted - versions are accepted because the value is interpolated into cache paths and - download URLs. - """ - raw = version if version is not None else os.environ.get("CLOAKBROWSER_VERSION") - if raw is None: - return None - normalized = raw.strip() - if not normalized: - return None - if not _VERSION_PIN_RE.fullmatch(normalized): - raise ValueError( - "Invalid browser version pin. Use a full numeric Chromium version, " - "e.g. '148.0.7778.215.2'." - ) - return normalized - - -def get_cache_dir() -> Path: - """Return the cache directory for downloaded binaries. - - Override with CLOAKBROWSER_CACHE_DIR env var. - Default: ~/.cloakbrowser/ - """ - custom = os.environ.get("CLOAKBROWSER_CACHE_DIR") - if custom: - return Path(custom) - return Path.home() / ".cloakbrowser" - - -def get_local_binary_override() -> str | None: - """Check if user has set a local binary path via env var. - - Set CLOAKBROWSER_BINARY_PATH to use a locally built Chromium instead of downloading. - """ - return os.environ.get("CLOAKBROWSER_BINARY_PATH") diff --git a/vendor/cloakbrowser/download.py b/vendor/cloakbrowser/download.py deleted file mode 100644 index 80766bd..0000000 --- a/vendor/cloakbrowser/download.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Binary resolution for cuttle. - -cuttle always runs against a baked local Chromium fork, selected via the -CLOAKBROWSER_BINARY_PATH env var. This module keeps only the local-override -path; the upstream download and binary-verification machinery is not vendored. -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -from .config import get_local_binary_override - -logger = logging.getLogger("cloakbrowser") - - -def ensure_binary(browser_version: str | None = None) -> str: - """Return the local Chromium binary path from CLOAKBROWSER_BINARY_PATH. - - cuttle always sets CLOAKBROWSER_BINARY_PATH to a baked fork binary. This - resolves it, erroring clearly if it is unset or points at a missing file. - """ - local_override = get_local_binary_override() - if not local_override: - raise RuntimeError( - "CLOAKBROWSER_BINARY_PATH is not set. cuttle ships no binary download; " - "point it at a local stealth Chromium build." - ) - path = Path(local_override) - if not path.exists(): - raise FileNotFoundError( - f"CLOAKBROWSER_BINARY_PATH set to '{local_override}' but file does not exist" - ) - logger.info("Using local binary override: %s", local_override) - return str(path) diff --git a/vendor/cloakbrowser/geoip.py b/vendor/cloakbrowser/geoip.py deleted file mode 100644 index 1d52885..0000000 --- a/vendor/cloakbrowser/geoip.py +++ /dev/null @@ -1,354 +0,0 @@ -"""GeoIP-based timezone and locale detection from proxy IP. - -Requires ``geoip2`` and ``socksio`` (both are cuttle runtime dependencies). -Downloads GeoLite2-City.mmdb (~70 MB) on first use, caches under the cache dir. -Background re-download after 30 days. -""" - -from __future__ import annotations - -import ipaddress -import logging -import math -import os -import socket -import tempfile -import threading -import time -from pathlib import Path -from urllib.parse import urlparse - -logger = logging.getLogger("cloakbrowser") - -# P3TERX mirror of MaxMind GeoLite2-City โ€” no license key needed -GEOIP_DB_URL = ( - "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb" -) -GEOIP_DB_FILENAME = "GeoLite2-City.mmdb" -GEOIP_UPDATE_INTERVAL = 30 * 86_400 # 30 days -DEFAULT_GEOIP_TIMEOUT_SECONDS = 5.0 -GEOIP_TIMEOUT_ENV = "CLOAKBROWSER_GEOIP_TIMEOUT_SECONDS" - -# Country ISO code โ†’ BCP 47 locale (covers ~90 % of proxy traffic) -COUNTRY_LOCALE_MAP: dict[str, str] = { - "US": "en-US", "GB": "en-GB", "AU": "en-AU", "CA": "en-CA", "NZ": "en-NZ", - "IE": "en-IE", "ZA": "en-ZA", "SG": "en-SG", - "DE": "de-DE", "AT": "de-AT", "CH": "de-CH", - "FR": "fr-FR", "BE": "fr-BE", - "ES": "es-ES", "MX": "es-MX", "AR": "es-AR", "CO": "es-CO", "CL": "es-CL", - "BR": "pt-BR", "PT": "pt-PT", - "IT": "it-IT", "NL": "nl-NL", - "JP": "ja-JP", "KR": "ko-KR", "CN": "zh-CN", "TW": "zh-TW", "HK": "zh-HK", - "RU": "ru-RU", "UA": "uk-UA", "PL": "pl-PL", "CZ": "cs-CZ", "RO": "ro-RO", - "IL": "he-IL", "TR": "tr-TR", "SA": "ar-SA", "AE": "ar-AE", "EG": "ar-EG", - "IN": "hi-IN", "ID": "id-ID", "PH": "en-PH", - "TH": "th-TH", "VN": "vi-VN", "MY": "ms-MY", - "SE": "sv-SE", "NO": "nb-NO", "DK": "da-DK", "FI": "fi-FI", - "GR": "el-GR", "HU": "hu-HU", "BG": "bg-BG", - # Extended coverage โ€” common residential/mobile proxy exits - "SI": "sl-SI", "SK": "sk-SK", "HR": "hr-HR", "RS": "sr-RS", "LT": "lt-LT", - "LV": "lv-LV", "EE": "et-EE", "IS": "is-IS", "LU": "fr-LU", "MT": "en-MT", - "CY": "el-CY", "MD": "ro-MD", "BY": "ru-BY", "GE": "ka-GE", "AL": "sq-AL", - "MK": "mk-MK", "BA": "bs-BA", - "PE": "es-PE", "VE": "es-VE", "EC": "es-EC", "UY": "es-UY", "CR": "es-CR", - "DO": "es-DO", "GT": "es-GT", "BO": "es-BO", "PY": "es-PY", - "PK": "en-PK", "BD": "bn-BD", "LK": "si-LK", "KZ": "ru-KZ", "IR": "fa-IR", - "IQ": "ar-IQ", "JO": "ar-JO", "LB": "ar-LB", "KW": "ar-KW", "QA": "ar-QA", - "OM": "ar-OM", "BH": "ar-BH", - "NG": "en-NG", "KE": "en-KE", "MA": "fr-MA", "DZ": "ar-DZ", "TN": "ar-TN", - "GH": "en-GH", - "AM": "hy-AM", "AZ": "az-AZ", "UZ": "uz-UZ", "KG": "ky-KG", "TJ": "tg-TJ", - "TM": "tk-TM", - "ME": "sr-ME", "XK": "sq-XK", "LI": "de-LI", "MC": "fr-MC", "AD": "ca-AD", - "MM": "my-MM", "KH": "km-KH", "LA": "lo-LA", "MN": "mn-MN", "BN": "ms-BN", - "MO": "zh-MO", - "YE": "ar-YE", "SY": "ar-SY", "PS": "ar-PS", "LY": "ar-LY", - "ET": "am-ET", "TZ": "sw-TZ", "UG": "en-UG", "SN": "fr-SN", "CI": "fr-CI", - "CM": "fr-CM", "AO": "pt-AO", "MZ": "pt-MZ", "ZM": "en-ZM", "ZW": "en-ZW", - "HN": "es-HN", "NI": "es-NI", "SV": "es-SV", "PA": "es-PA", "JM": "en-JM", - "TT": "en-TT", "PR": "es-PR", -} - - -def resolve_proxy_geo(proxy_url: str | None) -> tuple[str | None, str | None]: - """Resolve timezone and locale from a proxy's IP address. - - Returns ``(timezone, locale)`` โ€” either or both may be ``None`` on - failure (missing dep, DB download error, lookup miss). Never raises. - - When *proxy_url* is falsy, the machine's own public IP is used instead - (direct HTTP to the echo services, no proxy). - """ - tz, locale, _ip = resolve_proxy_geo_with_ip(proxy_url) - return tz, locale - - -def resolve_proxy_geo_with_ip( - proxy_url: str | None, -) -> tuple[str | None, str | None, str | None]: - """Resolve timezone, locale, and exit IP from a proxy. - - Returns ``(timezone, locale, exit_ip)``. The exit IP is a free bonus - from the lookup โ€” reused for WebRTC spoofing without an extra HTTP call. - - When *proxy_url* is falsy, the egress IP is the machine's own public IP - (echo services queried directly, no proxy), so geoip works proxy-free. - """ - try: - import geoip2.database # noqa: F811 - except ImportError: - raise ImportError("geoip2 is required for geoip=True") from None - - # Ensure the DB first โ€” the download must NOT be bounded by the resolution - # timeout (a first-use ~70MB fetch legitimately outlasts it). - db_path = _ensure_geoip_db() - - timeout = _get_geoip_timeout_seconds() - deadline = _deadline_from_timeout(timeout) - - # Exit IP (through proxy, or the machine's own public IP when proxy_url is - # falsy) is most accurate โ€” gateway DNS may differ from exit. Resolved even - # when the DB is unavailable: the IP does not need the DB, and dropping it on - # a DB hiccup would let WebRTC fall back to the real IP behind a proxy while - # the connection shows the proxy IP โ€” a real deanonymization. - ip = _resolve_exit_ip(proxy_url, timeout=_remaining_seconds(deadline)) - # Hostname fallback only applies to a proxy; no proxy โ†’ echo services only - if ip is None and proxy_url and not _deadline_expired(deadline): - ip = _resolve_proxy_ip(proxy_url) - if ip is None or _deadline_expired(deadline): - if deadline is not None and _deadline_expired(deadline): - logger.warning("GeoIP resolution timed out after %.1fs; continuing without GeoIP", timeout) - return None, None, None - - # DB only drives tz/locale; a missing/failed DB still returns the exit IP. - if db_path is None: - return None, None, ip - - try: - with geoip2.database.Reader(str(db_path)) as reader: - resp = reader.city(ip) - timezone = resp.location.time_zone - country = resp.country.iso_code - locale = COUNTRY_LOCALE_MAP.get(country) if country else None - logger.debug( - "GeoIP: %s โ†’ tz=%s, country=%s, locale=%s", - ip, timezone, country, locale, - ) - return timezone, locale, ip - except Exception as exc: - logger.warning("GeoIP lookup failed for %s: %s", ip, exc) - return None, None, ip - - -# --------------------------------------------------------------------------- -# Proxy IP resolution -# --------------------------------------------------------------------------- - - -def _resolve_proxy_ip(proxy_url: str) -> str | None: - """Extract proxy hostname from URL and resolve to an IP address.""" - try: - hostname = urlparse(proxy_url).hostname - if not hostname: - return None - - # Already a literal IP? - try: - socket.inet_pton(socket.AF_INET, hostname) - return hostname - except OSError: - pass - try: - socket.inet_pton(socket.AF_INET6, hostname) - return hostname - except OSError: - pass - - # DNS resolve (returns first result, handles both v4/v6) - results = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM) - if results: - ip = results[0][4][0] - logger.debug("Resolved proxy %s โ†’ %s", hostname, ip) - return ip - return None - except Exception as exc: - logger.warning("Failed to resolve proxy hostname: %s", exc) - return None - - -def _is_private_ip(ip: str) -> bool: - """Check if an IP address is private/internal (not routable on the internet).""" - try: - return ipaddress.ip_address(ip).is_private - except ValueError: - return False - - -# IP echo services โ€” fast, no auth, return just the IP -_IP_ECHO_URLS = [ - "https://api.ipify.org", - "https://checkip.amazonaws.com", - "https://ifconfig.me/ip", -] - - -def _get_geoip_timeout_seconds() -> float: - raw = os.getenv(GEOIP_TIMEOUT_ENV) - if not raw: - return DEFAULT_GEOIP_TIMEOUT_SECONDS - try: - timeout = float(raw) - except ValueError: - timeout = float("nan") - if not math.isfinite(timeout): - logger.warning( - "Invalid %s=%r; using %.1fs", - GEOIP_TIMEOUT_ENV, - raw, - DEFAULT_GEOIP_TIMEOUT_SECONDS, - ) - return DEFAULT_GEOIP_TIMEOUT_SECONDS - return max(timeout, 0.0) - - -def _deadline_from_timeout(timeout: float) -> float | None: - if timeout <= 0: - return None - return time.monotonic() + timeout - - -def _remaining_seconds(deadline: float | None) -> float | None: - if deadline is None: - return None - return max(deadline - time.monotonic(), 0.0) - - -def _deadline_expired(deadline: float | None) -> bool: - return deadline is not None and time.monotonic() >= deadline - - -def resolve_proxy_exit_ip(proxy_url: str | None) -> str | None: - """Resolve the egress IP, bounded by the GeoIP timeout. - - With a proxy this is the proxy's exit IP; with no proxy it is the - machine's own public IP (echo services queried directly). - """ - timeout = _get_geoip_timeout_seconds() - deadline = _deadline_from_timeout(timeout) - ip = _resolve_exit_ip(proxy_url, timeout=timeout) - if ip is None and _deadline_expired(deadline): - logger.warning("GeoIP resolution timed out after %.1fs; continuing without GeoIP", timeout) - return ip - - -def _resolve_exit_ip(proxy_url: str | None, timeout: float | None = None) -> str | None: - """Discover the egress IP via the echo services. - - Through *proxy_url* when given (the proxy's exit IP); directly (no proxy) - when *proxy_url* is falsy โ€” that returns the machine's own public IP. - """ - import httpx - - deadline = _deadline_from_timeout(timeout or 0) - - for url in _IP_ECHO_URLS: - try: - remaining = _remaining_seconds(deadline) - if remaining is not None and remaining <= 0: - return None - request_timeout = min(10.0, remaining) if remaining is not None else 10.0 - resp = httpx.get(url, proxy=proxy_url or None, timeout=request_timeout) - resp.raise_for_status() - ip = resp.text.strip() - # Validate it looks like an IP - ipaddress.ip_address(ip) - logger.debug("Exit IP via %s: %s", url, ip) - return ip - except httpx.UnsupportedProtocol: - logger.warning( - "SOCKS5 proxy requires the socksio package" - ) - return None - except Exception: - continue - logger.warning("Failed to discover exit IP through proxy") - return None - - -# --------------------------------------------------------------------------- -# GeoIP database management -# --------------------------------------------------------------------------- - - -def _get_geoip_dir() -> Path: - from .config import get_cache_dir - - return get_cache_dir() / "geoip" - - -def _ensure_geoip_db() -> Path | None: - """Return path to GeoLite2-City.mmdb, downloading on first use.""" - db_path = _get_geoip_dir() / GEOIP_DB_FILENAME - - if db_path.exists(): - _maybe_trigger_update(db_path) - return db_path - - try: - _download_geoip_db(db_path) - return db_path - except Exception as exc: - logger.warning("Failed to download GeoIP database: %s", exc) - return None - - -def _download_geoip_db(dest: Path) -> None: - """Atomic download of GeoLite2-City.mmdb via httpx.""" - import httpx - - dest.parent.mkdir(parents=True, exist_ok=True) - logger.info("Downloading GeoIP database (~70 MB) โ€ฆ") - - tmp_fd, tmp_name = tempfile.mkstemp(dir=dest.parent, suffix=".tmp") - tmp_path = Path(tmp_name) - try: - with httpx.stream( - "GET", GEOIP_DB_URL, follow_redirects=True, timeout=300.0 - ) as resp: - resp.raise_for_status() - total = int(resp.headers.get("content-length", 0)) - downloaded = 0 - last_pct = -1 - with open(tmp_fd, "wb") as f: - for chunk in resp.iter_bytes(chunk_size=65_536): - f.write(chunk) - downloaded += len(chunk) - if total: - pct = downloaded * 100 // total - if pct >= last_pct + 10: - last_pct = pct - logger.info("GeoIP download: %d %%", pct) - - tmp_path.rename(dest) - logger.info("GeoIP database ready: %s", dest) - except Exception: - tmp_path.unlink(missing_ok=True) - raise - - -def _maybe_trigger_update(db_path: Path) -> None: - """Re-download in background if DB is older than 30 days.""" - try: - age = time.time() - db_path.stat().st_mtime - if age < GEOIP_UPDATE_INTERVAL: - return - except OSError: - return - - def _bg() -> None: - try: - _download_geoip_db(db_path) - except Exception: - logger.debug("Background GeoIP update failed", exc_info=True) - - threading.Thread(target=_bg, daemon=True).start()