diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b27de0..515ba94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.25.x' + go-version: '1.26.x' - name: Download modules run: go mod download @@ -37,7 +37,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.25.x' + go-version: '1.26.x' - name: Start buildkitd run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..85e4ef2 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,54 @@ +name: Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install MkDocs + run: python -m pip install --upgrade pip mkdocs + + - name: Build docs site + run: mkdocs build --strict + + - name: Upload pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b6b0e3d..32f761b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: "1.25.x" + go-version: "1.26.x" - name: Build binary shell: bash diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..45ddf0a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +site/ diff --git a/AGENTS.md b/AGENTS.md index 301f37f..23b8456 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,3 +23,8 @@ - 2026-02-25: Integration build fixtures must be deterministic and self-contained (no external digest dependency) to avoid flaky CI. - 2026-02-25: Repository metadata is managed with GitHub CLI (`gh`) and should include concise description + discoverable topics. - 2026-02-25: Public release policy: publish prebuilt CLI artifacts for Linux/macOS/Windows on each published GitHub release. +- 2026-02-26: Added first-class build trace + graph workflows (`build --progress/--trace`, `graph --from`, `top --from`) powered by local JSONL trace artifacts. +- 2026-02-26: Formalized stable JSON envelope contract with `schemaVersion` and always-present `errors` array. +- 2026-02-26: Doctor UX now includes attempt trail, resolved backend details, paste-ready config snippet, and remediation guidance. +- 2026-02-26: Minimum required Go version is Go 1.26 for local builds and CI/release workflows. +- 2026-03-02: Documentation is published via MkDocs + GitHub Pages workflow (`.github/workflows/docs.yml`) with custom domain `docs.buildgraph.dev`. diff --git a/README.md b/README.md index 8080a29..7cb2bef 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,122 @@ # buildgraph -`buildgraph` is a Build Intelligence CLI focused on BuildKit-first workflows. +`buildgraph` is a Build Intelligence CLI for BuildKit-first workflows. -## Goals -- BuildKit-first build orchestration and diagnostics. -- Dockerfile intelligence across performance, cacheability, reproducibility, security, and policy. -- Human-first output with stable `--json` mode. -- Extensible backend architecture for future providers such as Buildah. -- SaaS-ready foundations (auth/events/capabilities) with opt-in behavior. +## 30-Second Quickstart -## Install +### Direct BuildKit socket + +```bash +go build ./cmd/buildgraph + +./buildgraph build \ + --context integration/fixtures \ + --file Dockerfile.integration \ + --output local \ + --local-dest /tmp/buildgraph-out \ + --endpoint unix:///run/buildkit/buildkitd.sock \ + --progress=json \ + --trace /tmp/buildgraph.trace.jsonl + +./buildgraph graph --from /tmp/buildgraph.trace.jsonl --format dot --output /tmp/buildgraph.dot +./buildgraph top --from /tmp/buildgraph.trace.jsonl +``` + +### Docker Desktop / Docker Engine + +```bash +go build ./cmd/buildgraph + +./buildgraph build \ + --context integration/fixtures \ + --file Dockerfile.integration \ + --image-ref buildgraph/quickstart:dev \ + --progress=human \ + --trace ./buildgraph.trace.jsonl + +./buildgraph graph --from ./buildgraph.trace.jsonl --format json +./buildgraph top --from ./buildgraph.trace.jsonl --limit 5 +``` + +## Example Output (Sample) + +```text +$ buildgraph top --from ./buildgraph.trace.jsonl +Vertices analyzed: 9 + +Slowest vertices: +1. 4823 ms RUN apk add --no-cache build-base (sha256:...) +2. 2111 ms RUN go build ./cmd/buildgraph (sha256:...) + +Critical path: 7034 ms +1. FROM golang:1.26-alpine (112 ms) +2. RUN apk add --no-cache build-base (4823 ms) +3. RUN go build ./cmd/buildgraph (2111 ms) +``` + +## Commands + +```bash +buildgraph analyze [--context .] [--file Dockerfile] [--severity-threshold low|medium|high|critical] [--fail-on policy|security|any] [--json] +buildgraph build [--context .] [--file Dockerfile] [--target NAME] [--platform linux/amd64] [--build-arg KEY=VALUE] [--secret id=foo,src=./foo.txt] [--output image|oci|local] [--image-ref REF] [--oci-dest PATH] [--local-dest PATH] [--backend auto|buildkit] [--endpoint URL] [--progress human|json|none] [--trace out.jsonl] [--json] +buildgraph graph --from out.jsonl [--format dot|svg|json] [--output PATH] [--json] +buildgraph top --from out.jsonl [--limit N] [--json] +buildgraph backend list +buildgraph doctor +buildgraph auth login --user --token +buildgraph auth logout +buildgraph auth whoami +buildgraph config show +buildgraph version +``` + +## JSON Output Contract (`--json`) + +All machine-readable command output uses a versioned envelope: + +```json +{ + "apiVersion": "buildgraph.dev/v1", + "command": "build", + "schemaVersion": "1", + "timestamp": "2026-02-26T00:00:00Z", + "durationMs": 1234, + "result": {}, + "errors": [] +} +``` + +## Rule Documentation + +Rule pages backing finding links are tracked in this repository: + +- [Rules Index](./docs/rules/index.md) + +Docs are published from `docs/` using the GitHub Actions workflow: + +- [docs.yml](./.github/workflows/docs.yml) + +## What Data Is Collected + +`buildgraph` stores local state in a SQLite database to support diagnostics and history: +- run metadata (`command`, duration, exit code, success/failure) +- analysis findings +- build result metadata +- local events + +`buildgraph` can also write local build traces (`--trace`) as JSONL. + +## What Is Never Uploaded By Default + +- no build context files are uploaded by default +- no findings/build metadata are uploaded by default +- no telemetry is sent unless explicitly enabled (`telemetry.enabled: true`) + +Auth credentials are stored locally via OS keyring when available, with local file fallback. + +## Install from Source + +Requires Go 1.26+. ```bash go build ./cmd/buildgraph @@ -45,20 +152,6 @@ Invoke-WebRequest -Uri "https://github.com/Makepad-fr/buildgraph/releases/latest Expand-Archive -Path ".\\buildgraph_windows_amd64.zip" -DestinationPath ".\\buildgraph" ``` -## Commands - -```bash -buildgraph analyze [--context .] [--file Dockerfile] [--severity-threshold low|medium|high|critical] [--fail-on policy|security|any] [--json] -buildgraph build [--context .] [--file Dockerfile] [--target NAME] [--platform linux/amd64] [--build-arg KEY=VALUE] [--secret id=foo,src=./foo.txt] [--output image|oci|local] [--image-ref REF] [--oci-dest PATH] [--local-dest PATH] [--backend auto|buildkit] [--endpoint URL] [--json] -buildgraph backend list -buildgraph doctor -buildgraph auth login --user --token -buildgraph auth logout -buildgraph auth whoami -buildgraph config show -buildgraph version -``` - ## Configuration Default merge precedence: @@ -98,5 +191,6 @@ go test ./... ``` ## Notes -- Build execution avoids shelling out to external build commands. -- Docker-backed mode currently supports image export, while direct BuildKit mode supports image/OCI/local exports. + +- Build execution avoids shelling out to Docker/BuildKit CLIs. +- Docker-backed mode supports image export; direct BuildKit mode supports image/OCI/local exports. diff --git a/docs/CNAME b/docs/CNAME new file mode 100644 index 0000000..a293359 --- /dev/null +++ b/docs/CNAME @@ -0,0 +1 @@ +docs.buildgraph.dev diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c8e9951 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,14 @@ +# Buildgraph Documentation + +This directory contains the source content for the public documentation URLs used by `buildgraph` findings. + +## Rules + +- [Rules Index](./rules/index.md) + +Each rule page is named after its rule ID, for example: +- `docs/rules/BG_REPRO_FROM_MUTABLE.md` +- `docs/rules/BG_SEC_ROOT_USER.md` + +These pages are designed to map directly to public URLs under: +- `https://docs.buildgraph.dev/rules/` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..324e958 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,18 @@ +# Buildgraph Documentation + +Buildgraph publishes rule-level guidance for findings reported by `buildgraph analyze`. + +## Primary Sections + +- [Rules Overview](./rules/index.md) + +## Public Rule URLs + +Finding links are designed to resolve under: + +- `https://docs.buildgraph.dev/rules/` + +Examples: + +- `https://docs.buildgraph.dev/rules/BG_REPRO_FROM_MUTABLE` +- `https://docs.buildgraph.dev/rules/BG_SEC_ROOT_USER` diff --git a/docs/rules/BG_CACHE_ARG_LATE.md b/docs/rules/BG_CACHE_ARG_LATE.md new file mode 100644 index 0000000..e608c10 --- /dev/null +++ b/docs/rules/BG_CACHE_ARG_LATE.md @@ -0,0 +1,36 @@ +# BG_CACHE_ARG_LATE + +- Dimension: `cacheability` +- Severity: `medium` + +## Summary + +`ARG` values are declared late, after expensive build steps. + +## Why It Matters + +Changing late `ARG` values can invalidate large portions of the build graph. + +## Typical Trigger + +```dockerfile +RUN make deps +ARG APP_VERSION +RUN make build +``` + +## Recommended Fix + +Declare stable `ARG` values before expensive operations. + +```dockerfile +ARG APP_VERSION +RUN make deps +RUN make build +``` + +## Remediation Checklist + +- Move stable args earlier. +- Keep volatile args scoped only where needed. +- Re-check cache hit rate in CI. diff --git a/docs/rules/BG_CACHE_COPY_ALL_EARLY.md b/docs/rules/BG_CACHE_COPY_ALL_EARLY.md new file mode 100644 index 0000000..07c732b --- /dev/null +++ b/docs/rules/BG_CACHE_COPY_ALL_EARLY.md @@ -0,0 +1,35 @@ +# BG_CACHE_COPY_ALL_EARLY + +- Dimension: `cacheability` +- Severity: `high` + +## Summary + +A broad `COPY .` happens before dependency install steps. + +## Why It Matters + +Any source file change invalidates dependency cache and forces expensive reinstall/build steps. + +## Typical Trigger + +```dockerfile +COPY . . +RUN npm ci +``` + +## Recommended Fix + +Copy dependency manifests first, install dependencies, then copy the remaining source. + +```dockerfile +COPY package.json package-lock.json ./ +RUN npm ci +COPY . . +``` + +## Remediation Checklist + +- Move lockfiles/manifests before install steps. +- Keep app source copy after dependency restore. +- Add `.dockerignore` to reduce noisy context changes. diff --git a/docs/rules/BG_PERF_APT_SPLIT.md b/docs/rules/BG_PERF_APT_SPLIT.md new file mode 100644 index 0000000..631a0dd --- /dev/null +++ b/docs/rules/BG_PERF_APT_SPLIT.md @@ -0,0 +1,35 @@ +# BG_PERF_APT_SPLIT + +- Dimension: `performance` +- Severity: `medium` + +## Summary + +`apt-get update` is executed without an install in the same layer. + +## Why It Matters + +Splitting update and install into separate layers increases rebuild time and can use stale package indexes. + +## Typical Trigger + +```dockerfile +RUN apt-get update +RUN apt-get install -y curl +``` + +## Recommended Fix + +Combine update and install in one instruction and clear apt lists: + +```dockerfile +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* +``` + +## Remediation Checklist + +- Merge `apt-get update` with related install commands. +- Add `--no-install-recommends` where possible. +- Remove apt cache in the same layer. diff --git a/docs/rules/BG_PERF_TOO_MANY_RUN.md b/docs/rules/BG_PERF_TOO_MANY_RUN.md new file mode 100644 index 0000000..9bb7425 --- /dev/null +++ b/docs/rules/BG_PERF_TOO_MANY_RUN.md @@ -0,0 +1,31 @@ +# BG_PERF_TOO_MANY_RUN + +- Dimension: `performance` +- Severity: `low` + +## Summary + +The Dockerfile has many `RUN` instructions, increasing layer count. + +## Why It Matters + +Too many layers can slow builds and make images harder to reason about. + +## Typical Trigger + +Many small `RUN` instructions that could be grouped. + +## Recommended Fix + +Consolidate related commands while keeping readability and cache boundaries sensible. + +```dockerfile +RUN apk add --no-cache bash curl git \ + && update-ca-certificates +``` + +## Remediation Checklist + +- Merge only logically related steps. +- Keep package install and cleanup in the same layer. +- Avoid over-consolidating unrelated steps. diff --git a/docs/rules/BG_POL_MISSING_HEALTHCHECK.md b/docs/rules/BG_POL_MISSING_HEALTHCHECK.md new file mode 100644 index 0000000..6891aec --- /dev/null +++ b/docs/rules/BG_POL_MISSING_HEALTHCHECK.md @@ -0,0 +1,31 @@ +# BG_POL_MISSING_HEALTHCHECK + +- Dimension: `policy` +- Severity: `low` + +## Summary + +The Dockerfile has no `HEALTHCHECK`. + +## Why It Matters + +Schedulers cannot distinguish healthy from unhealthy containers as reliably. + +## Typical Trigger + +No `HEALTHCHECK` instruction in the final image stage. + +## Recommended Fix + +Define a lightweight health probe. + +```dockerfile +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD curl -fsS http://127.0.0.1:8080/health || exit 1 +``` + +## Remediation Checklist + +- Use an application-level readiness endpoint. +- Keep probe fast and deterministic. +- Tune interval/timeout/retries for production behavior. diff --git a/docs/rules/BG_POL_MISSING_SOURCE_LABEL.md b/docs/rules/BG_POL_MISSING_SOURCE_LABEL.md new file mode 100644 index 0000000..2cac80f --- /dev/null +++ b/docs/rules/BG_POL_MISSING_SOURCE_LABEL.md @@ -0,0 +1,30 @@ +# BG_POL_MISSING_SOURCE_LABEL + +- Dimension: `policy` +- Severity: `medium` + +## Summary + +The image does not include `org.opencontainers.image.source`. + +## Why It Matters + +Missing provenance metadata reduces traceability and compliance reporting quality. + +## Typical Trigger + +No OCI source label is present. + +## Recommended Fix + +Set the source repository label in the final image stage. + +```dockerfile +LABEL org.opencontainers.image.source="https://github.com/acme/service" +``` + +## Remediation Checklist + +- Add OCI source label to final stage. +- Keep label aligned with canonical repository URL. +- Add additional OCI labels where useful (revision, created, version). diff --git a/docs/rules/BG_REPRO_APT_UNPINNED.md b/docs/rules/BG_REPRO_APT_UNPINNED.md new file mode 100644 index 0000000..3c8f088 --- /dev/null +++ b/docs/rules/BG_REPRO_APT_UNPINNED.md @@ -0,0 +1,34 @@ +# BG_REPRO_APT_UNPINNED + +- Dimension: `reproducibility` +- Severity: `medium` + +## Summary + +`apt-get install` installs packages without explicit versions. + +## Why It Matters + +Package versions can change across mirror updates, producing different images over time. + +## Typical Trigger + +```dockerfile +RUN apt-get update && apt-get install -y curl ca-certificates +``` + +## Recommended Fix + +Pin versions where practical, or use a controlled artifact/mirror policy. + +```dockerfile +RUN apt-get update \ + && apt-get install -y curl=8.5.0-2ubuntu10 ca-certificates=20240203 \ + && rm -rf /var/lib/apt/lists/* +``` + +## Remediation Checklist + +- Pin critical packages. +- Align with your distro release cadence. +- Use internal mirrors for strict reproducibility. diff --git a/docs/rules/BG_REPRO_FROM_MUTABLE.md b/docs/rules/BG_REPRO_FROM_MUTABLE.md new file mode 100644 index 0000000..a4a28d5 --- /dev/null +++ b/docs/rules/BG_REPRO_FROM_MUTABLE.md @@ -0,0 +1,32 @@ +# BG_REPRO_FROM_MUTABLE + +- Dimension: `reproducibility` +- Severity: `high` + +## Summary + +A `FROM` image uses a mutable tag and is not pinned to a digest. + +## Why It Matters + +Mutable tags can drift over time, causing non-deterministic builds. + +## Typical Trigger + +```dockerfile +FROM alpine:3.20 +``` + +## Recommended Fix + +Pin images to immutable digests. + +```dockerfile +FROM alpine:3.20@sha256: +``` + +## Remediation Checklist + +- Pin all stage base images. +- Automate digest refresh using a scheduled dependency workflow. +- Keep tag plus digest for readability. diff --git a/docs/rules/BG_SEC_CURL_PIPE_SH.md b/docs/rules/BG_SEC_CURL_PIPE_SH.md new file mode 100644 index 0000000..9711c89 --- /dev/null +++ b/docs/rules/BG_SEC_CURL_PIPE_SH.md @@ -0,0 +1,34 @@ +# BG_SEC_CURL_PIPE_SH + +- Dimension: `security` +- Severity: `critical` + +## Summary + +Remote scripts are piped directly into a shell. + +## Why It Matters + +This bypasses integrity verification and allows remote tampering to execute immediately. + +## Typical Trigger + +```dockerfile +RUN curl -fsSL https://example.com/install.sh | sh +``` + +## Recommended Fix + +Download, verify checksum/signature, then execute. + +```dockerfile +RUN curl -fsSLo /tmp/install.sh https://example.com/install.sh \ + && echo " /tmp/install.sh" | sha256sum -c - \ + && sh /tmp/install.sh +``` + +## Remediation Checklist + +- Never pipe unverified remote scripts to shell. +- Verify integrity before execution. +- Prefer signed release assets when available. diff --git a/docs/rules/BG_SEC_PLAIN_SECRET_ENV.md b/docs/rules/BG_SEC_PLAIN_SECRET_ENV.md new file mode 100644 index 0000000..aec63cd --- /dev/null +++ b/docs/rules/BG_SEC_PLAIN_SECRET_ENV.md @@ -0,0 +1,34 @@ +# BG_SEC_PLAIN_SECRET_ENV + +- Dimension: `security` +- Severity: `critical` + +## Summary + +A likely secret is stored with `ENV`, embedding it into image metadata/history. + +## Why It Matters + +Secrets become recoverable from image layers, registry metadata, and build logs. + +## Typical Trigger + +```dockerfile +ENV AWS_SECRET_ACCESS_KEY=... +``` + +## Recommended Fix + +Use BuildKit secret mounts at build time, and runtime secret injection for containers. + +```dockerfile +# build: --secret id=npm_token,src=.npm_token +RUN --mount=type=secret,id=npm_token \ + NPM_TOKEN="$(cat /run/secrets/npm_token)" npm ci +``` + +## Remediation Checklist + +- Remove secrets from `ENV` and `ARG` where possible. +- Use BuildKit `--secret` for build-time credentials. +- Rotate any leaked credentials. diff --git a/docs/rules/BG_SEC_ROOT_USER.md b/docs/rules/BG_SEC_ROOT_USER.md new file mode 100644 index 0000000..772ebe9 --- /dev/null +++ b/docs/rules/BG_SEC_ROOT_USER.md @@ -0,0 +1,31 @@ +# BG_SEC_ROOT_USER + +- Dimension: `security` +- Severity: `high` + +## Summary + +The resulting image runs as root. + +## Why It Matters + +Running as root increases impact of runtime compromise. + +## Typical Trigger + +No `USER` instruction, or `USER root` in final stage. + +## Recommended Fix + +Create a dedicated runtime user and switch to it in the final stage. + +```dockerfile +RUN addgroup --system app && adduser --system --ingroup app app +USER app +``` + +## Remediation Checklist + +- Set `USER` in the final runtime stage. +- Ensure runtime paths are writable by that user. +- Keep root-only operations in build stages. diff --git a/docs/rules/index.md b/docs/rules/index.md new file mode 100644 index 0000000..bf9795c --- /dev/null +++ b/docs/rules/index.md @@ -0,0 +1,29 @@ +# Buildgraph Rule Reference + +This index maps every built-in rule ID to its documentation page. + +## Performance + +- [BG_PERF_APT_SPLIT](./BG_PERF_APT_SPLIT.md) +- [BG_PERF_TOO_MANY_RUN](./BG_PERF_TOO_MANY_RUN.md) + +## Cacheability + +- [BG_CACHE_COPY_ALL_EARLY](./BG_CACHE_COPY_ALL_EARLY.md) +- [BG_CACHE_ARG_LATE](./BG_CACHE_ARG_LATE.md) + +## Reproducibility + +- [BG_REPRO_FROM_MUTABLE](./BG_REPRO_FROM_MUTABLE.md) +- [BG_REPRO_APT_UNPINNED](./BG_REPRO_APT_UNPINNED.md) + +## Security + +- [BG_SEC_ROOT_USER](./BG_SEC_ROOT_USER.md) +- [BG_SEC_CURL_PIPE_SH](./BG_SEC_CURL_PIPE_SH.md) +- [BG_SEC_PLAIN_SECRET_ENV](./BG_SEC_PLAIN_SECRET_ENV.md) + +## Policy + +- [BG_POL_MISSING_SOURCE_LABEL](./BG_POL_MISSING_SOURCE_LABEL.md) +- [BG_POL_MISSING_HEALTHCHECK](./BG_POL_MISSING_HEALTHCHECK.md) diff --git a/go.mod b/go.mod index 5954e75..e7d4208 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,14 @@ module github.com/Makepad-fr/buildgraph -go 1.25.0 +go 1.26.0 require ( github.com/containerd/platforms v1.0.0-rc.2 + github.com/docker/cli v29.1.4+incompatible github.com/moby/buildkit v0.27.1 github.com/moby/moby/api v1.53.0 github.com/moby/moby/client v0.2.2 + github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 github.com/zalando/go-keyring v0.2.6 golang.org/x/sync v0.19.0 @@ -17,6 +19,7 @@ require ( require ( al.essio.dev/pkg/shellescape v1.5.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/containerd/console v1.0.5 // indirect github.com/containerd/containerd/api v1.10.0 // indirect github.com/containerd/containerd/v2 v2.2.1 // indirect github.com/containerd/continuity v0.4.5 // indirect @@ -25,8 +28,9 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.7 // indirect github.com/containerd/typeurl/v2 v2.2.3 // indirect - github.com/danieljoos/wincred v1.2.2 // indirect + github.com/danieljoos/wincred v1.2.3 // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker-credential-helpers v0.9.5 // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -41,6 +45,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/in-toto/in-toto-golang v0.9.0 // indirect github.com/klauspost/compress v1.18.3 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -48,8 +53,8 @@ require ( github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/signal v0.7.1 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect @@ -57,7 +62,9 @@ require ( github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/tonistiigi/fsutil v0.0.0-20251211185533-a2aa163d723f // indirect + github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 // indirect github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect + github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0 // indirect @@ -73,6 +80,7 @@ require ( golang.org/x/net v0.48.0 // indirect golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.32.0 // indirect + golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251103181224-f26f9409b101 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect google.golang.org/grpc v1.76.0 // indirect diff --git a/go.sum b/go.sum index 8e3d46d..eee86cb 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUo github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= github.com/containerd/cgroups/v3 v3.1.2 h1:OSosXMtkhI6Qove637tg1XgK4q+DhR0mX8Wi8EhrHa4= github.com/containerd/cgroups/v3 v3.1.2/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= +github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc= +github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= github.com/containerd/containerd/api v1.10.0/go.mod h1:NBm1OAk8ZL+LG8R0ceObGxT5hbUYj7CzTmR3xh0DlMM= github.com/containerd/containerd/v2 v2.2.1 h1:TpyxcY4AL5A+07dxETevunVS5zxqzuq7ZqJXknM11yk= @@ -43,8 +45,9 @@ github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++ github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is= github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= -github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= -github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= +github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= +github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -86,6 +89,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/in-toto/in-toto-golang v0.9.0 h1:tHny7ac4KgtsfrG6ybU8gVOZux2H8jN05AXJ9EBM1XU= @@ -122,6 +127,8 @@ github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -158,8 +165,10 @@ github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg= github.com/spdx/tools-golang v0.5.7/go.mod h1:jg7w0LOpoNAw6OxKEzCoqPC2GCTj45LyTlVmXubDsYw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tonistiigi/fsutil v0.0.0-20251211185533-a2aa163d723f h1:Z4NEQ86qFl1mHuCu9gwcE+EYCwDKfXAYXZbdIXyxmEA= @@ -168,6 +177,8 @@ github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 h1:2f304B10 github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= +github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Qb2z5hI1UHxSQt4JMyxebFR15KnApw= +github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -225,6 +236,7 @@ golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= @@ -234,6 +246,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/internal/analyze/rules_docs_test.go b/internal/analyze/rules_docs_test.go new file mode 100644 index 0000000..86e21c7 --- /dev/null +++ b/internal/analyze/rules_docs_test.go @@ -0,0 +1,45 @@ +package analyze + +import ( + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" +) + +func TestBuiltinRulesHaveDocsPages(t *testing.T) { + t.Parallel() + + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatalf("resolve current file for docs path") + } + repoRoot := filepath.Clean(filepath.Join(filepath.Dir(currentFile), "..", "..")) + docsDir := filepath.Join(repoRoot, "docs", "rules") + + rules := BuiltinRules() + missing := make([]string, 0) + for id, rule := range rules { + expectedURL := docsBase + id + if rule.DocsRef != expectedURL { + t.Fatalf("unexpected docs url for %s: got=%q want=%q", id, rule.DocsRef, expectedURL) + } + + docPath := filepath.Join(docsDir, id+".md") + content, err := os.ReadFile(docPath) + if err != nil { + missing = append(missing, id) + continue + } + if !strings.Contains(string(content), "# "+id) { + t.Fatalf("docs page %s must include heading '# %s'", docPath, id) + } + } + + if len(missing) > 0 { + sort.Strings(missing) + t.Fatalf("missing docs pages for rules: %s", strings.Join(missing, ", ")) + } +} diff --git a/internal/backend/backend.go b/internal/backend/backend.go index cd1675a..5dbd7b2 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -119,9 +119,19 @@ type DetectResult struct { Mode string `json:"mode"` Available bool `json:"available"` Details string `json:"details"` + Attempts []DetectAttempt `json:"attempts,omitempty"` Metadata map[string]string `json:"metadata"` } +type DetectAttempt struct { + Source string `json:"source"` + Endpoint string `json:"endpoint"` + Mode string `json:"mode"` + Status string `json:"status"` + Details string `json:"details,omitempty"` + Error string `json:"error,omitempty"` +} + type BackendCapabilities struct { SupportsAnalyze bool `json:"supportsAnalyze"` SupportsImageOutput bool `json:"supportsImageOutput"` @@ -132,11 +142,16 @@ type BackendCapabilities struct { } type BuildProgressEvent struct { - Timestamp time.Time `json:"timestamp"` - Phase string `json:"phase"` - Message string `json:"message"` - VertexID string `json:"vertexId,omitempty"` - Status string `json:"status,omitempty"` + Timestamp time.Time `json:"timestamp"` + Phase string `json:"phase"` + Message string `json:"message"` + VertexID string `json:"vertexId,omitempty"` + Inputs []string `json:"inputs,omitempty"` + Status string `json:"status,omitempty"` + Started *time.Time `json:"started,omitempty"` + Completed *time.Time `json:"completed,omitempty"` + Cached bool `json:"cached,omitempty"` + Error string `json:"error,omitempty"` } type BuildProgressFunc func(BuildProgressEvent) diff --git a/internal/backend/buildkit/backend.go b/internal/backend/buildkit/backend.go index 3947f0e..04bd6ee 100644 --- a/internal/backend/buildkit/backend.go +++ b/internal/backend/buildkit/backend.go @@ -16,8 +16,18 @@ const BackendName = "buildkit" type Backend struct { analyzer *analyze.Engine - direct *DirectDriver - docker *DockerDriver + direct directClient + docker dockerClient +} + +type directClient interface { + Ping(ctx context.Context, endpoint string) error + Build(ctx context.Context, endpoint string, req backend.BuildRequest, progress backend.BuildProgressFunc) (backend.BuildResult, error) +} + +type dockerClient interface { + Ping(ctx context.Context) error + Build(ctx context.Context, req backend.BuildRequest, progress backend.BuildProgressFunc) (backend.BuildResult, error) } func NewBackend() *Backend { @@ -41,6 +51,8 @@ func (b *Backend) Detect(ctx context.Context, req backend.DetectRequest) (backen Available: false, Mode: "none", Details: err.Error(), + Attempts: resolved.Attempts, + Metadata: map[string]string{}, }, err } @@ -50,6 +62,7 @@ func (b *Backend) Detect(ctx context.Context, req backend.DetectRequest) (backen Mode: resolved.Mode, Available: true, Details: resolved.Details, + Attempts: resolved.Attempts, Metadata: map[string]string{ "resolutionSource": resolved.Source, }, @@ -135,54 +148,101 @@ type endpointResolution struct { Mode string Source string Details string + Attempts []backend.DetectAttempt } func (b *Backend) resolveEndpoint(ctx context.Context, explicit, projectConfigPath, globalConfigPath string) (endpointResolution, error) { - if endpoint := strings.TrimSpace(explicit); endpoint != "" { - if err := b.direct.Ping(ctx, endpoint); err == nil { - return endpointResolution{Endpoint: endpoint, Mode: "direct", Source: "flag", Details: "using explicit BuildKit endpoint"}, nil + attempts := make([]backend.DetectAttempt, 0, 8) + + tryDirect := func(source, endpoint, details string) (endpointResolution, bool) { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return endpointResolution{}, false + } + attempt := backend.DetectAttempt{ + Source: source, + Endpoint: endpoint, + Mode: "direct", + Details: details, + } + if err := b.direct.Ping(ctx, endpoint); err != nil { + attempt.Status = "error" + attempt.Error = err.Error() + attempts = append(attempts, attempt) + return endpointResolution{}, false } + attempt.Status = "ok" + attempts = append(attempts, attempt) + return endpointResolution{ + Endpoint: endpoint, + Mode: "direct", + Source: source, + Details: details, + Attempts: attempts, + }, true } - if endpoint := strings.TrimSpace(os.Getenv("BUILDKIT_HOST")); endpoint != "" { - if err := b.direct.Ping(ctx, endpoint); err == nil { - return endpointResolution{Endpoint: endpoint, Mode: "direct", Source: "env", Details: "using BUILDKIT_HOST"}, nil + tryDocker := func(source, details string) (endpointResolution, bool) { + attempt := backend.DetectAttempt{ + Source: source, + Endpoint: "docker://local", + Mode: "docker", + Details: details, } + if err := b.docker.Ping(ctx); err != nil { + attempt.Status = "error" + attempt.Error = err.Error() + attempts = append(attempts, attempt) + return endpointResolution{}, false + } + attempt.Status = "ok" + attempts = append(attempts, attempt) + return endpointResolution{ + Endpoint: "docker://local", + Mode: "docker", + Source: source, + Details: details, + Attempts: attempts, + }, true } - if endpoint := readEndpointFromConfig(projectConfigPath); endpoint != "" { - if err := b.direct.Ping(ctx, endpoint); err == nil { - return endpointResolution{Endpoint: endpoint, Mode: "direct", Source: "project-config", Details: "using project config endpoint"}, nil - } + if resolved, ok := tryDirect("flag", explicit, "using explicit BuildKit endpoint"); ok { + return resolved, nil } - if endpoint := readEndpointFromConfig(globalConfigPath); endpoint != "" { - if err := b.direct.Ping(ctx, endpoint); err == nil { - return endpointResolution{Endpoint: endpoint, Mode: "direct", Source: "global-config", Details: "using global config endpoint"}, nil - } + + if resolved, ok := tryDirect("env", os.Getenv("BUILDKIT_HOST"), "using BUILDKIT_HOST"); ok { + return resolved, nil + } + + if resolved, ok := tryDirect("project-config", readEndpointFromConfig(projectConfigPath), "using project config endpoint"); ok { + return resolved, nil + } + if resolved, ok := tryDirect("global-config", readEndpointFromConfig(globalConfigPath), "using global config endpoint"); ok { + return resolved, nil } if runtime.GOOS == "windows" { - if err := b.docker.Ping(ctx); err == nil { - return endpointResolution{Endpoint: "docker://local", Mode: "docker", Source: "auto", Details: "docker daemon reachable"}, nil + if resolved, ok := tryDocker("auto", "docker daemon reachable"); ok { + return resolved, nil } for _, endpoint := range windowsDefaultEndpoints() { - if err := b.direct.Ping(ctx, endpoint); err == nil { - return endpointResolution{Endpoint: endpoint, Mode: "direct", Source: "auto", Details: "direct BuildKit endpoint discovered"}, nil + if resolved, ok := tryDirect("auto", endpoint, "direct BuildKit endpoint discovered"); ok { + return resolved, nil } } - return endpointResolution{}, fmt.Errorf("no BuildKit endpoint detected on Windows: docker daemon unavailable and no direct endpoint reachable") + return endpointResolution{Attempts: attempts}, fmt.Errorf("no BuildKit endpoint detected on Windows: docker daemon unavailable and no direct endpoint reachable") } for _, endpoint := range unixDefaultEndpoints() { - if err := b.direct.Ping(ctx, endpoint); err == nil { - return endpointResolution{Endpoint: endpoint, Mode: "direct", Source: "auto", Details: "direct BuildKit endpoint discovered"}, nil + if resolved, ok := tryDirect("auto", endpoint, "direct BuildKit endpoint discovered"); ok { + return resolved, nil } } - if err := b.docker.Ping(ctx); err == nil { - return endpointResolution{Endpoint: "docker://local", Mode: "docker", Source: "auto", Details: "docker daemon reachable"}, nil + if resolved, ok := tryDocker("auto", "docker daemon reachable"); ok { + return resolved, nil } - return endpointResolution{}, fmt.Errorf("no BuildKit endpoint detected") + return endpointResolution{Attempts: attempts}, fmt.Errorf("no BuildKit endpoint detected") } func readEndpointFromConfig(path string) string { diff --git a/internal/backend/buildkit/backend_test.go b/internal/backend/buildkit/backend_test.go new file mode 100644 index 0000000..448f5ed --- /dev/null +++ b/internal/backend/buildkit/backend_test.go @@ -0,0 +1,113 @@ +package buildkit + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/Makepad-fr/buildgraph/internal/backend" +) + +type fakeDirect struct { + ping map[string]error +} + +func (f *fakeDirect) Ping(_ context.Context, endpoint string) error { + if f.ping == nil { + return fmt.Errorf("ping failed: %s", endpoint) + } + if err, ok := f.ping[endpoint]; ok { + return err + } + return fmt.Errorf("ping failed: %s", endpoint) +} + +func (f *fakeDirect) Build(context.Context, string, backend.BuildRequest, backend.BuildProgressFunc) (backend.BuildResult, error) { + return backend.BuildResult{}, nil +} + +type fakeDocker struct { + pingErr error +} + +func (f *fakeDocker) Ping(context.Context) error { + if f.pingErr != nil { + return f.pingErr + } + return nil +} + +func (f *fakeDocker) Build(context.Context, backend.BuildRequest, backend.BuildProgressFunc) (backend.BuildResult, error) { + return backend.BuildResult{}, nil +} + +func TestResolveEndpointRecordsAttemptsInPriorityOrder(t *testing.T) { + t.Setenv("BUILDKIT_HOST", "unix:///env.sock") + + be := &Backend{ + direct: &fakeDirect{ + ping: map[string]error{ + "unix:///flag.sock": errors.New("flag unavailable"), + "unix:///env.sock": nil, + }, + }, + docker: &fakeDocker{pingErr: errors.New("docker unavailable")}, + } + + resolved, err := be.resolveEndpoint(context.Background(), "unix:///flag.sock", "", "") + if err != nil { + t.Fatalf("resolve endpoint: %v", err) + } + if got, want := resolved.Source, "env"; got != want { + t.Fatalf("unexpected source: got=%q want=%q", got, want) + } + if got, want := resolved.Endpoint, "unix:///env.sock"; got != want { + t.Fatalf("unexpected endpoint: got=%q want=%q", got, want) + } + if len(resolved.Attempts) < 2 { + t.Fatalf("expected at least 2 attempts, got %d", len(resolved.Attempts)) + } + if got, want := resolved.Attempts[0].Source, "flag"; got != want { + t.Fatalf("unexpected first source: got=%q want=%q", got, want) + } + if got, want := resolved.Attempts[1].Source, "env"; got != want { + t.Fatalf("unexpected second source: got=%q want=%q", got, want) + } + if got, want := resolved.Attempts[0].Status, "error"; got != want { + t.Fatalf("unexpected flag attempt status: got=%q want=%q", got, want) + } + if got, want := resolved.Attempts[1].Status, "ok"; got != want { + t.Fatalf("unexpected env attempt status: got=%q want=%q", got, want) + } +} + +func TestDetectReturnsAttemptTrailOnFailure(t *testing.T) { + be := &Backend{ + direct: &fakeDirect{ + ping: map[string]error{ + "unix:///flag.sock": errors.New("permission denied"), + }, + }, + docker: &fakeDocker{pingErr: errors.New("docker unavailable")}, + } + + result, err := be.Detect(context.Background(), backend.DetectRequest{ + Endpoint: "unix:///flag.sock", + }) + if err == nil { + t.Fatalf("expected detect error") + } + if result.Available { + t.Fatalf("expected detect availability to be false") + } + if len(result.Attempts) == 0 { + t.Fatalf("expected detect attempts to be recorded") + } + if got, want := result.Attempts[0].Source, "flag"; got != want { + t.Fatalf("unexpected first attempt source: got=%q want=%q", got, want) + } + if got, want := result.Attempts[0].Status, "error"; got != want { + t.Fatalf("unexpected first attempt status: got=%q want=%q", got, want) + } +} diff --git a/internal/backend/buildkit/direct.go b/internal/backend/buildkit/direct.go index 12554bf..1aef0a5 100644 --- a/internal/backend/buildkit/direct.go +++ b/internal/backend/buildkit/direct.go @@ -118,23 +118,13 @@ func (d *DirectDriver) Build(ctx context.Context, endpoint string, req backend.B if vertex == nil { continue } - state := "running" - if vertex.Completed != nil { - state = "completed" - } if vertex.Cached { cacheHits++ } else { cacheMisses++ } if progressFn != nil { - progressFn(backend.BuildProgressEvent{ - Timestamp: time.Now().UTC(), - Phase: "build", - Message: vertex.Name, - VertexID: vertex.Digest.String(), - Status: state, - }) + progressFn(toBuildProgressEvent(vertex)) } } } @@ -171,6 +161,43 @@ func (d *DirectDriver) Build(ctx context.Context, endpoint string, req backend.B }, nil } +func cloneTimePtr(value *time.Time) *time.Time { + if value == nil { + return nil + } + cloned := value.UTC() + return &cloned +} + +func toBuildProgressEvent(vertex *bkclient.Vertex) backend.BuildProgressEvent { + state := "running" + if vertex != nil && vertex.Completed != nil { + state = "completed" + } + + event := backend.BuildProgressEvent{ + Timestamp: time.Now().UTC(), + Phase: "build", + Status: state, + } + if vertex == nil { + return event + } + + inputs := make([]string, 0, len(vertex.Inputs)) + for _, input := range vertex.Inputs { + inputs = append(inputs, input.String()) + } + event.Message = vertex.Name + event.VertexID = vertex.Digest.String() + event.Inputs = inputs + event.Started = cloneTimePtr(vertex.Started) + event.Completed = cloneTimePtr(vertex.Completed) + event.Cached = vertex.Cached + event.Error = vertex.Error + return event +} + func configureExports(solveOpt *bkclient.SolveOpt, req backend.BuildRequest) ([]string, []string, error) { switch req.OutputMode { case "", backend.OutputImage: diff --git a/internal/backend/buildkit/direct_test.go b/internal/backend/buildkit/direct_test.go index 96b40dc..df6bcaf 100644 --- a/internal/backend/buildkit/direct_test.go +++ b/internal/backend/buildkit/direct_test.go @@ -2,9 +2,11 @@ package buildkit import ( "testing" + "time" "github.com/Makepad-fr/buildgraph/internal/backend" bkclient "github.com/moby/buildkit/client" + "github.com/opencontainers/go-digest" ) func TestConfigureExportsLocalUsesOutputDir(t *testing.T) { @@ -38,3 +40,42 @@ func TestConfigureExportsLocalUsesOutputDir(t *testing.T) { t.Fatalf("expected Attrs to be nil for local exporter, got %v", export.Attrs) } } + +func TestToBuildProgressEventMapsGraphFields(t *testing.T) { + t.Parallel() + + started := time.Unix(10, 0).UTC() + completed := time.Unix(12, 0).UTC() + vertex := &bkclient.Vertex{ + Digest: digest.FromString("vertex"), + Inputs: []digest.Digest{digest.FromString("input-a"), digest.FromString("input-b")}, + Name: "RUN apk add curl", + Started: &started, + Completed: &completed, + Cached: true, + Error: "boom", + } + + event := toBuildProgressEvent(vertex) + if got, want := event.VertexID, vertex.Digest.String(); got != want { + t.Fatalf("unexpected vertex id: got=%q want=%q", got, want) + } + if got, want := len(event.Inputs), 2; got != want { + t.Fatalf("unexpected input count: got=%d want=%d", got, want) + } + if event.Started == nil || !event.Started.Equal(started) { + t.Fatalf("unexpected started time: %v", event.Started) + } + if event.Completed == nil || !event.Completed.Equal(completed) { + t.Fatalf("unexpected completed time: %v", event.Completed) + } + if !event.Cached { + t.Fatalf("expected cached=true") + } + if got, want := event.Error, "boom"; got != want { + t.Fatalf("unexpected error: got=%q want=%q", got, want) + } + if got, want := event.Status, "completed"; got != want { + t.Fatalf("unexpected status: got=%q want=%q", got, want) + } +} diff --git a/internal/backend/buildkit/dockerdriver.go b/internal/backend/buildkit/dockerdriver.go index c505b90..5900ff4 100644 --- a/internal/backend/buildkit/dockerdriver.go +++ b/internal/backend/buildkit/dockerdriver.go @@ -4,9 +4,11 @@ import ( "archive/tar" "context" "encoding/json" + "errors" "fmt" "io" "io/fs" + "net" "os" "path/filepath" "strings" @@ -14,9 +16,13 @@ import ( "github.com/Makepad-fr/buildgraph/internal/backend" ctrplatforms "github.com/containerd/platforms" + dockerconfig "github.com/docker/cli/cli/config" + bksession "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/auth/authprovider" buildtypes "github.com/moby/moby/api/types/build" dockerclient "github.com/moby/moby/client" specs "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" ) type DockerDriver struct{} @@ -96,6 +102,13 @@ func (d *DockerDriver) Build(ctx context.Context, req backend.BuildRequest, prog Version: buildtypes.BuilderBuildKit, Remove: true, } + session, err := startDockerBuildSession(ctx, cli) + if err != nil { + return backend.BuildResult{}, err + } + defer session.Close() + options.SessionID = session.ID() + if len(req.Platforms) > 0 { options.Platforms = make([]specs.Platform, 0, len(req.Platforms)) for _, platform := range req.Platforms { @@ -222,3 +235,64 @@ func tarContextDirectory(root string) (io.ReadCloser, error) { return pr, nil } + +type dockerBuildSession struct { + session *bksession.Session + cancel context.CancelFunc + group *errgroup.Group +} + +func startDockerBuildSession(ctx context.Context, cli *dockerclient.Client) (*dockerBuildSession, error) { + // BuildKit frontend resolution on the daemon expects a live session; wire it + // through Docker's /session hijack endpoint. + session, err := bksession.NewSession(ctx, "buildgraph") + if err != nil { + return nil, fmt.Errorf("create docker build session: %w", err) + } + dockerCfg := dockerconfig.LoadDefaultConfigFile(io.Discard) + session.Allow(authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{ + AuthConfigProvider: authprovider.LoadAuthConfig(dockerCfg), + })) + runCtx, cancel := context.WithCancel(ctx) + group, groupCtx := errgroup.WithContext(runCtx) + group.Go(func() error { + return session.Run(groupCtx, func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { + return cli.DialHijack(ctx, "/session", proto, meta) + }) + }) + return &dockerBuildSession{ + session: session, + cancel: cancel, + group: group, + }, nil +} + +func (s *dockerBuildSession) ID() string { + if s == nil || s.session == nil { + return "" + } + return s.session.ID() +} + +func (s *dockerBuildSession) Close() error { + if s == nil { + return nil + } + if s.cancel != nil { + s.cancel() + } + if s.session != nil { + _ = s.session.Close() + } + if s.group == nil { + return nil + } + err := s.group.Wait() + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) { + return nil + } + return err +} diff --git a/internal/cli/app.go b/internal/cli/app.go index 46c21bc..890e524 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -20,6 +20,7 @@ import ( "github.com/Makepad-fr/buildgraph/internal/platform/capabilities" "github.com/Makepad-fr/buildgraph/internal/platform/events" "github.com/Makepad-fr/buildgraph/internal/state" + "github.com/Makepad-fr/buildgraph/internal/trace" "github.com/Makepad-fr/buildgraph/internal/version" ) @@ -123,6 +124,10 @@ func (a *App) Run(ctx context.Context, args []string) int { } case "backend": exitCode, runErr = a.runBackend(global, cmdArgs) + case "graph": + exitCode, runErr = a.runGraph(global, cmdArgs) + case "top": + exitCode, runErr = a.runTop(global, cmdArgs) case "doctor": exitCode, runErr = a.runDoctor(ctx, global, loadedCfg, stateStore) case "auth": @@ -166,6 +171,8 @@ func (a *App) Run(ctx context.Context, args []string) int { } func (a *App) runAnalyze(ctx context.Context, global GlobalOptions, loaded config.Loaded, args []string) (backend.AnalyzeResult, []backend.Finding, int, error) { + startedAt := time.Now().UTC() + allowed, err := a.capabilities.Has(ctx, capabilities.FeatureAnalyze) if err != nil { return backend.AnalyzeResult{}, nil, ExitAuthDenied, err @@ -212,19 +219,17 @@ func (a *App) runAnalyze(ctx context.Context, global GlobalOptions, loaded confi failErr := fmt.Errorf("analysis found violations matching fail-on=%s", *failOn) if global.JSON { - env := output.Envelope{ - APIVersion: output.APIVersion, - Command: "analyze", - Timestamp: time.Now().UTC(), - DurationMS: 0, - Result: result, - } + errors := []output.ErrorItem{} if failure { - env.Errors = []output.ErrorItem{{Code: "violation", Message: failErr.Error()}} + errors = append(errors, output.ErrorItem{Code: "violation", Message: failErr.Error()}) + } + if err := output.WriteJSON(a.io.Out, output.NewEnvelope("analyze", startedAt, result, errors)); err != nil { + return backend.AnalyzeResult{}, nil, ExitInternal, err } - _ = output.WriteJSON(a.io.Out, env) } else { - _ = output.WriteAnalyze(a.io.Out, result) + if err := output.WriteAnalyze(a.io.Out, result); err != nil { + return backend.AnalyzeResult{}, nil, ExitInternal, err + } } if failure { @@ -238,6 +243,8 @@ func (a *App) runAnalyze(ctx context.Context, global GlobalOptions, loaded confi } func (a *App) runBuild(ctx context.Context, global GlobalOptions, loaded config.Loaded, args []string) (*backend.BuildResult, int, error) { + startedAt := time.Now().UTC() + allowed, err := a.capabilities.Has(ctx, capabilities.FeatureBuild) if err != nil { return nil, ExitAuthDenied, err @@ -261,6 +268,8 @@ func (a *App) runBuild(ctx context.Context, global GlobalOptions, loaded config. localDest := fs.String("local-dest", "", "Destination directory for local output") backendName := fs.String("backend", loaded.Config.Backend, "Backend selector") endpoint := fs.String("endpoint", loaded.Config.Endpoint, "BuildKit endpoint") + progressModeRaw := fs.String("progress", "auto", "Progress mode: human|json|none") + tracePath := fs.String("trace", "", "Write build trace to JSONL path") fs.Var(&platforms, "platform", "Target platform (repeatable)") fs.Var(&buildArgs, "build-arg", "Build arg key=value (repeatable)") @@ -275,11 +284,41 @@ func (a *App) runBuild(ctx context.Context, global GlobalOptions, loaded config. return nil, ExitBackend, err } + progressMode, err := normalizeProgressMode(*progressModeRaw, global.JSON) + if err != nil { + return nil, ExitUsage, err + } + + var traceFile io.Closer + var traceWriter *trace.Writer + if path := strings.TrimSpace(*tracePath); path != "" { + file, writer, err := trace.OpenFileWriter(path) + if err != nil { + return nil, ExitConfigState, err + } + traceFile = file + traceWriter = writer + defer traceFile.Close() + } + + var stderrTraceWriter *trace.Writer + if progressMode == "json" { + stderrTraceWriter = trace.NewWriter(a.io.Err) + } + progress := func(event backend.BuildProgressEvent) { - if global.JSON { - return + record := trace.ProgressRecord("build", event) + if traceWriter != nil { + _ = traceWriter.WriteRecord(record) + } + switch progressMode { + case "human": + fmt.Fprintf(a.io.Err, "[%s] %s\n", event.Phase, strings.TrimSpace(event.Message)) + case "json": + if stderrTraceWriter != nil { + _ = stderrTraceWriter.WriteRecord(record) + } } - fmt.Fprintf(a.io.Err, "[%s] %s\n", event.Phase, strings.TrimSpace(event.Message)) } result, err := selectedBackend.Build(ctx, backend.BuildRequest{ @@ -299,24 +338,30 @@ func (a *App) runBuild(ctx context.Context, global GlobalOptions, loaded config. GlobalConfigPath: loaded.Paths.GlobalPath, }, progress) if err != nil { + if traceWriter != nil { + _ = traceWriter.WriteRecord(trace.FailureRecord("build", "build_failed", err.Error())) + } return nil, ExitBuildFailed, err } + if traceWriter != nil { + _ = traceWriter.WriteRecord(trace.ResultRecord("build", result)) + } if global.JSON { - _ = output.WriteJSON(a.io.Out, output.Envelope{ - APIVersion: output.APIVersion, - Command: "build", - Timestamp: time.Now().UTC(), - DurationMS: 0, - Result: result, - }) + if err := output.WriteJSON(a.io.Out, output.NewEnvelope("build", startedAt, result, nil)); err != nil { + return nil, ExitInternal, err + } } else { - _ = output.WriteBuild(a.io.Out, result) + if err := output.WriteBuild(a.io.Out, result); err != nil { + return nil, ExitInternal, err + } } return &result, ExitOK, nil } func (a *App) runBackend(global GlobalOptions, args []string) (int, error) { + startedAt := time.Now().UTC() + if len(args) == 0 { return ExitUsage, fmt.Errorf("backend subcommand is required") } @@ -325,15 +370,9 @@ func (a *App) runBackend(global GlobalOptions, args []string) (int, error) { } names := a.registry.List() if global.JSON { - return ExitOK, output.WriteJSON(a.io.Out, output.Envelope{ - APIVersion: output.APIVersion, - Command: "backend list", - Timestamp: time.Now().UTC(), - DurationMS: 0, - Result: map[string]any{ - "backends": names, - }, - }) + return ExitOK, output.WriteJSON(a.io.Out, output.NewEnvelope("backend list", startedAt, map[string]any{ + "backends": names, + }, nil)) } for _, name := range names { fmt.Fprintln(a.io.Out, name) @@ -342,64 +381,82 @@ func (a *App) runBackend(global GlobalOptions, args []string) (int, error) { } func (a *App) runDoctor(ctx context.Context, global GlobalOptions, loaded config.Loaded, store *state.Store) (int, error) { + startedAt := time.Now().UTC() + checks := map[string]string{ "config.global": status(loaded.Paths.GlobalExists, loaded.Paths.GlobalPath), "config.project": status(loaded.Paths.ProjectExists, loaded.Paths.ProjectPath), } + hasError := false if store != nil { checks["state.sqlite"] = "ok: " + store.Path() } else { checks["state.sqlite"] = "error: unavailable" + hasError = true + } + + detect := backend.DetectResult{ + Backend: buildkit.BackendName, + Mode: "none", + Details: "backend detection was not executed", + Metadata: map[string]string{}, } selectedBackend, err := a.resolveBackend(loaded.Config.Backend) if err != nil { checks["backend.detect"] = "error: " + err.Error() + detect.Details = err.Error() + hasError = true } else { - detect, detectErr := selectedBackend.Detect(ctx, backend.DetectRequest{ + detect, err = selectedBackend.Detect(ctx, backend.DetectRequest{ Backend: loaded.Config.Backend, Endpoint: loaded.Config.Endpoint, ProjectConfigPath: loaded.Paths.ProjectPath, GlobalConfigPath: loaded.Paths.GlobalPath, }) - if detectErr != nil { - checks["backend.detect"] = "error: " + detectErr.Error() + if err != nil { + checks["backend.detect"] = "error: " + err.Error() + hasError = true } else { checks["backend.detect"] = fmt.Sprintf("ok: mode=%s endpoint=%s", detect.Mode, detect.Endpoint) } } + report := output.DoctorReport{ + Checks: checks, + Attempts: detect.Attempts, + Found: detect, + ConfigSnippet: doctorConfigSnippet(loaded.Config.Endpoint, detect.Endpoint), + CommonFixes: doctorCommonFixes(loaded.Config.Endpoint, detect), + } + if global.JSON { - if err := output.WriteJSON(a.io.Out, output.Envelope{ - APIVersion: output.APIVersion, - Command: "doctor", - Timestamp: time.Now().UTC(), - DurationMS: 0, - Result: map[string]any{ - "checks": checks, - }, - }); err != nil { + errors := []output.ErrorItem{} + if hasError { + errors = append(errors, output.ErrorItem{Code: "doctor_failed", Message: "doctor detected failing checks"}) + } + if err := output.WriteJSON(a.io.Out, output.NewEnvelope("doctor", startedAt, report, errors)); err != nil { return ExitInternal, err } } else { - if err := output.WriteDoctor(a.io.Out, checks); err != nil { + if err := output.WriteDoctor(a.io.Out, report); err != nil { return ExitInternal, err } } - for _, value := range checks { - if strings.HasPrefix(value, "error:") { - err := fmt.Errorf("doctor detected failing checks") - if global.JSON { - return ExitBackend, markReportedError(err) - } - return ExitBackend, err + if hasError { + err := fmt.Errorf("doctor detected failing checks") + if global.JSON { + return ExitBackend, markReportedError(err) } + return ExitBackend, err } return ExitOK, nil } func (a *App) runAuth(global GlobalOptions, args []string) (int, error) { + startedAt := time.Now().UTC() + if len(args) == 0 { return ExitUsage, fmt.Errorf("auth subcommand is required") } @@ -426,35 +483,31 @@ func (a *App) runAuth(global GlobalOptions, args []string) (int, error) { if err := manager.Save(auth.Credentials{User: *user, Token: *token}); err != nil { return ExitConfigState, err } - return a.writeSimple(global.JSON, "auth login", map[string]any{"status": "logged-in", "user": *user}) + return a.writeSimple(global.JSON, "auth login", startedAt, map[string]any{"status": "logged-in", "user": *user}) case "logout": if err := manager.Delete(); err != nil { return ExitConfigState, err } - return a.writeSimple(global.JSON, "auth logout", map[string]any{"status": "logged-out"}) + return a.writeSimple(global.JSON, "auth logout", startedAt, map[string]any{"status": "logged-out"}) case "whoami": creds, err := manager.Load() if err != nil { return ExitAuthDenied, fmt.Errorf("not logged in") } - return a.writeSimple(global.JSON, "auth whoami", map[string]any{"user": creds.User, "source": creds.Source, "storedAt": creds.StoredAt}) + return a.writeSimple(global.JSON, "auth whoami", startedAt, map[string]any{"user": creds.User, "source": creds.Source, "storedAt": creds.StoredAt}) default: return ExitUsage, fmt.Errorf("unsupported auth subcommand %q", subcommand) } } func (a *App) runConfig(global GlobalOptions, loaded config.Loaded, args []string) (int, error) { + startedAt := time.Now().UTC() + if len(args) == 0 || args[0] != "show" { return ExitUsage, fmt.Errorf("supported config command: show") } if global.JSON { - return ExitOK, output.WriteJSON(a.io.Out, output.Envelope{ - APIVersion: output.APIVersion, - Command: "config show", - Timestamp: time.Now().UTC(), - DurationMS: 0, - Result: loaded, - }) + return ExitOK, output.WriteJSON(a.io.Out, output.NewEnvelope("config show", startedAt, loaded, nil)) } fmt.Fprintf(a.io.Out, "Global config: %s\n", loaded.Paths.GlobalPath) @@ -466,23 +519,19 @@ func (a *App) runConfig(global GlobalOptions, loaded config.Loaded, args []strin } func (a *App) runVersion(global GlobalOptions) (int, error) { + startedAt := time.Now().UTC() + payload := map[string]any{ "version": version.Version, "commit": version.Commit, "buildDate": version.BuildDate, } - return a.writeSimple(global.JSON, "version", payload) + return a.writeSimple(global.JSON, "version", startedAt, payload) } -func (a *App) writeSimple(asJSON bool, command string, result any) (int, error) { +func (a *App) writeSimple(asJSON bool, command string, startedAt time.Time, result any) (int, error) { if asJSON { - if err := output.WriteJSON(a.io.Out, output.Envelope{ - APIVersion: output.APIVersion, - Command: command, - Timestamp: time.Now().UTC(), - DurationMS: 0, - Result: result, - }); err != nil { + if err := output.WriteJSON(a.io.Out, output.NewEnvelope(command, startedAt, result, nil)); err != nil { return ExitInternal, err } } else { @@ -679,6 +728,54 @@ func status(ok bool, detail string) string { return "missing: " + detail } +func normalizeProgressMode(value string, globalJSON bool) (string, error) { + mode := strings.ToLower(strings.TrimSpace(value)) + switch mode { + case "", "auto": + if globalJSON { + return "none", nil + } + return "human", nil + case "human", "json", "none": + return mode, nil + default: + return "", fmt.Errorf("invalid --progress %q (expected human|json|none)", value) + } +} + +func doctorConfigSnippet(configEndpoint, detectedEndpoint string) string { + endpoint := strings.TrimSpace(detectedEndpoint) + if endpoint == "" { + endpoint = strings.TrimSpace(configEndpoint) + } + if endpoint == "" { + endpoint = "unix:///run/buildkit/buildkitd.sock" + } + return fmt.Sprintf("backend: buildkit\nendpoint: %q\n", endpoint) +} + +func doctorCommonFixes(configEndpoint string, detect backend.DetectResult) []string { + fixes := []string{ + "If using a direct BuildKit socket, verify buildkitd is running and your user can access the socket path.", + "If using Docker Desktop/Engine, confirm the daemon is reachable and the active Docker context matches your expected environment.", + "If endpoint detection is wrong, pin backend and endpoint in .buildgraph.yaml to avoid ambiguous auto-detection.", + } + + if strings.TrimSpace(configEndpoint) != "" { + fixes = append(fixes, fmt.Sprintf("Configured endpoint is %q; remove conflicting BUILDGRAPH_ENDPOINT or BUILDKIT_HOST values if they should not override config.", configEndpoint)) + } + if source := detect.Metadata["resolutionSource"]; source == "env" { + fixes = append(fixes, "Detection resolved from environment; clear BUILDKIT_HOST if this endpoint is stale.") + } + for _, attempt := range detect.Attempts { + if strings.Contains(strings.ToLower(attempt.Error), "permission denied") { + fixes = append(fixes, "Permission denied was reported while probing BuildKit. Add your user to the required group or adjust socket ACLs.") + break + } + } + return fixes +} + type stringSliceFlag []string func (s *stringSliceFlag) String() string { @@ -751,16 +848,10 @@ func (a *App) writeError(asJSON bool, command string, durationMs int64, err erro return } if asJSON { - _ = output.WriteJSON(a.io.Err, output.Envelope{ - APIVersion: output.APIVersion, - Command: command, - Timestamp: time.Now().UTC(), - DurationMS: durationMs, - Errors: []output.ErrorItem{{ - Code: "error", - Message: err.Error(), - }}, - }) + _ = output.WriteJSON(a.io.Err, output.NewEnvelopeWithDuration(command, durationMs, nil, []output.ErrorItem{{ + Code: "error", + Message: err.Error(), + }})) return } fmt.Fprintf(a.io.Err, "Error: %v\n", err) @@ -774,7 +865,9 @@ func (a *App) printHelp(w io.Writer) { fmt.Fprintln(w, "") fmt.Fprintln(w, "Commands:") fmt.Fprintln(w, " analyze Analyze Dockerfile and build context") - fmt.Fprintln(w, " build Execute BuildKit build") + fmt.Fprintln(w, " build Execute BuildKit build (--progress, --trace)") + fmt.Fprintln(w, " graph Build graph artifact from trace (--from, --format, --output)") + fmt.Fprintln(w, " top Show slowest vertices and critical path from trace") fmt.Fprintln(w, " backend list List available backends") fmt.Fprintln(w, " doctor Run environment diagnostics") fmt.Fprintln(w, " auth Manage SaaS authentication state") diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index fae4817..aa7cdc5 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -32,3 +32,40 @@ func TestParseGlobalFlagsFromCommandTail(t *testing.T) { t.Fatalf("unexpected remaining args: %v", remaining) } } + +func TestNormalizeProgressMode(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + value string + globalJSON bool + want string + wantErr bool + }{ + {name: "auto human", value: "auto", globalJSON: false, want: "human"}, + {name: "auto json", value: "auto", globalJSON: true, want: "none"}, + {name: "explicit json", value: "json", globalJSON: false, want: "json"}, + {name: "invalid", value: "wat", globalJSON: false, wantErr: true}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := normalizeProgressMode(tc.value, tc.globalJSON) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error") + } + return + } + if err != nil { + t.Fatalf("normalize progress mode: %v", err) + } + if got != tc.want { + t.Fatalf("unexpected mode: got=%q want=%q", got, tc.want) + } + }) + } +} diff --git a/internal/cli/graph.go b/internal/cli/graph.go new file mode 100644 index 0000000..36020b5 --- /dev/null +++ b/internal/cli/graph.go @@ -0,0 +1,234 @@ +package cli + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/Makepad-fr/buildgraph/internal/output" + "github.com/Makepad-fr/buildgraph/internal/trace" +) + +func (a *App) runGraph(global GlobalOptions, args []string) (int, error) { + startedAt := time.Now().UTC() + + fs := flag.NewFlagSet("graph", flag.ContinueOnError) + fs.SetOutput(io.Discard) + + from := fs.String("from", "", "Trace JSONL path") + format := fs.String("format", "dot", "Graph format: dot|svg|json") + outputPath := fs.String("output", "", "Write output to file") + + if err := fs.Parse(args); err != nil { + return ExitUsage, err + } + if strings.TrimSpace(*from) == "" { + return ExitUsage, fmt.Errorf("--from is required") + } + + renderFormat := strings.ToLower(strings.TrimSpace(*format)) + switch renderFormat { + case "dot", "svg", "json": + default: + return ExitUsage, fmt.Errorf("unsupported graph format %q (expected dot|svg|json)", renderFormat) + } + if renderFormat == "svg" && strings.TrimSpace(*outputPath) == "" { + return ExitUsage, fmt.Errorf("--output is required for --format=svg") + } + + records, err := trace.LoadFile(*from) + if err != nil { + return ExitConfigState, err + } + graph, err := trace.BuildGraph(records) + if err != nil { + if errors.Is(err, trace.ErrNoVertexData) { + return ExitBackend, err + } + return ExitInternal, err + } + + content, err := renderGraph(graph, renderFormat) + if err != nil { + return ExitInternal, err + } + + if path := strings.TrimSpace(*outputPath); path != "" { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return ExitConfigState, fmt.Errorf("create output dir: %w", err) + } + if err := os.WriteFile(path, content, 0o644); err != nil { + return ExitConfigState, fmt.Errorf("write output file: %w", err) + } + } + + if global.JSON { + result := map[string]any{ + "from": *from, + "format": renderFormat, + "vertexCount": len(graph.Vertices), + "edgeCount": len(graph.Edges), + "outputPath": strings.TrimSpace(*outputPath), + } + if strings.TrimSpace(*outputPath) == "" { + switch renderFormat { + case "json": + result["graph"] = graph + default: + result["content"] = string(content) + } + } + if err := output.WriteJSON(a.io.Out, output.NewEnvelope("graph", startedAt, result, nil)); err != nil { + return ExitInternal, err + } + return ExitOK, nil + } + + if path := strings.TrimSpace(*outputPath); path != "" { + _, _ = fmt.Fprintf(a.io.Out, "Wrote graph output to %s\n", path) + return ExitOK, nil + } + _, _ = a.io.Out.Write(content) + if len(content) == 0 || content[len(content)-1] != '\n' { + _, _ = fmt.Fprintln(a.io.Out) + } + return ExitOK, nil +} + +func (a *App) runTop(global GlobalOptions, args []string) (int, error) { + startedAt := time.Now().UTC() + + fs := flag.NewFlagSet("top", flag.ContinueOnError) + fs.SetOutput(io.Discard) + + from := fs.String("from", "", "Trace JSONL path") + limit := fs.Int("limit", 10, "Maximum rows for slowest vertices") + + if err := fs.Parse(args); err != nil { + return ExitUsage, err + } + if strings.TrimSpace(*from) == "" { + return ExitUsage, fmt.Errorf("--from is required") + } + if *limit <= 0 { + return ExitUsage, fmt.Errorf("--limit must be greater than 0") + } + + records, err := trace.LoadFile(*from) + if err != nil { + return ExitConfigState, err + } + graph, err := trace.BuildGraph(records) + if err != nil { + if errors.Is(err, trace.ErrNoVertexData) { + return ExitBackend, err + } + return ExitInternal, err + } + result := trace.AnalyzeTop(graph, *limit) + + if global.JSON { + payload := map[string]any{ + "from": *from, + "top": result, + } + if err := output.WriteJSON(a.io.Out, output.NewEnvelope("top", startedAt, payload, nil)); err != nil { + return ExitInternal, err + } + return ExitOK, nil + } + + if err := writeTopHuman(a.io.Out, result); err != nil { + return ExitInternal, err + } + return ExitOK, nil +} + +func renderGraph(graph trace.Graph, format string) ([]byte, error) { + switch format { + case "dot": + return []byte(trace.DOT(graph)), nil + case "json": + payload, err := json.MarshalIndent(graph, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal graph json: %w", err) + } + return append(payload, '\n'), nil + case "svg": + return renderDOTAsSVG(trace.DOT(graph)) + default: + return nil, fmt.Errorf("unsupported graph format %q", format) + } +} + +func renderDOTAsSVG(dotSource string) ([]byte, error) { + cmd := exec.Command("dot", "-Tsvg") + cmd.Stdin = strings.NewReader(dotSource) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + if errors.Is(err, exec.ErrNotFound) { + return nil, fmt.Errorf("graphviz 'dot' is required for --format=svg. install Graphviz and retry, or use --format=dot") + } + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = err.Error() + } + return nil, fmt.Errorf("render svg with graphviz: %s", msg) + } + return stdout.Bytes(), nil +} + +func writeTopHuman(w io.Writer, result trace.TopResult) error { + if _, err := fmt.Fprintf(w, "Vertices analyzed: %d\n", result.VertexCount); err != nil { + return err + } + + if _, err := fmt.Fprintln(w, "\nSlowest vertices:"); err != nil { + return err + } + if len(result.Slowest) == 0 { + if _, err := fmt.Fprintln(w, "none"); err != nil { + return err + } + } else { + for i, row := range result.Slowest { + if _, err := fmt.Fprintf(w, "%d. %d ms %s (%s)\n", i+1, row.DurationMS, row.Name, row.ID); err != nil { + return err + } + } + } + + if _, err := fmt.Fprintf(w, "\nCritical path: %d ms\n", result.CriticalPath.DurationMS); err != nil { + return err + } + if len(result.CriticalPath.Vertices) == 0 { + if _, err := fmt.Fprintln(w, "none"); err != nil { + return err + } + } else { + for i, row := range result.CriticalPath.Vertices { + if _, err := fmt.Fprintf(w, "%d. %s (%d ms)\n", i+1, row.Name, row.DurationMS); err != nil { + return err + } + } + } + + if result.HasCycle { + if _, err := fmt.Fprintf(w, "\nWarning: %s\n", result.CycleDetected); err != nil { + return err + } + } + return nil +} diff --git a/internal/cli/graph_test.go b/internal/cli/graph_test.go new file mode 100644 index 0000000..7c5a89b --- /dev/null +++ b/internal/cli/graph_test.go @@ -0,0 +1,229 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Makepad-fr/buildgraph/internal/backend" + "github.com/Makepad-fr/buildgraph/internal/config" + "github.com/Makepad-fr/buildgraph/internal/output" + "github.com/Makepad-fr/buildgraph/internal/trace" +) + +func TestRunGraphDotAndJSON(t *testing.T) { + t.Parallel() + + tracePath := writeTraceFixture(t) + + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + app, err := NewApp(IO{In: strings.NewReader(""), Out: stdout, Err: stderr}) + if err != nil { + t.Fatalf("create app: %v", err) + } + + code, err := app.runGraph(GlobalOptions{}, []string{"--from", tracePath, "--format", "dot"}) + if err != nil { + t.Fatalf("run graph dot: %v", err) + } + if code != ExitOK { + t.Fatalf("unexpected exit code: %d", code) + } + if !strings.Contains(stdout.String(), "digraph buildgraph") { + t.Fatalf("dot output missing graph header: %s", stdout.String()) + } + + stdout.Reset() + code, err = app.runGraph(GlobalOptions{}, []string{"--from", tracePath, "--format", "json"}) + if err != nil { + t.Fatalf("run graph json: %v", err) + } + if code != ExitOK { + t.Fatalf("unexpected exit code: %d", code) + } + + var graph trace.Graph + if err := json.Unmarshal(stdout.Bytes(), &graph); err != nil { + t.Fatalf("decode graph json: %v", err) + } + if len(graph.Vertices) == 0 { + t.Fatalf("expected graph vertices in json output") + } +} + +func TestRunTopPrintsCriticalPath(t *testing.T) { + t.Parallel() + + tracePath := writeTraceFixture(t) + stdout := &bytes.Buffer{} + app, err := NewApp(IO{In: strings.NewReader(""), Out: stdout, Err: &bytes.Buffer{}}) + if err != nil { + t.Fatalf("create app: %v", err) + } + + code, err := app.runTop(GlobalOptions{}, []string{"--from", tracePath, "--limit", "2"}) + if err != nil { + t.Fatalf("run top: %v", err) + } + if code != ExitOK { + t.Fatalf("unexpected exit code: %d", code) + } + text := stdout.String() + if !strings.Contains(text, "Slowest vertices:") { + t.Fatalf("top output missing slowest section: %s", text) + } + if !strings.Contains(text, "Critical path:") { + t.Fatalf("top output missing critical path section: %s", text) + } +} + +func TestRunGraphSVGRequiresGraphviz(t *testing.T) { + tracePath := writeTraceFixture(t) + t.Setenv("PATH", t.TempDir()) + + app, err := NewApp(IO{In: strings.NewReader(""), Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}) + if err != nil { + t.Fatalf("create app: %v", err) + } + + code, err := app.runGraph(GlobalOptions{}, []string{ + "--from", tracePath, + "--format", "svg", + "--output", filepath.Join(t.TempDir(), "graph.svg"), + }) + if err == nil { + t.Fatalf("expected graphviz error") + } + if code != ExitInternal { + t.Fatalf("unexpected exit code: %d", code) + } + if !strings.Contains(err.Error(), "graphviz 'dot' is required") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestRunBuildRejectsInvalidProgressMode(t *testing.T) { + t.Parallel() + + app, err := NewApp(IO{In: strings.NewReader(""), Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}) + if err != nil { + t.Fatalf("create app: %v", err) + } + + loaded := config.Loaded{ + Config: config.DefaultConfig(), + } + _, code, err := app.runBuild(context.Background(), GlobalOptions{}, loaded, []string{"--progress", "invalid"}) + if err == nil { + t.Fatalf("expected invalid progress error") + } + if code != ExitUsage { + t.Fatalf("unexpected exit code: %d", code) + } +} + +func TestRunDoctorHumanOutputHasRemediationSections(t *testing.T) { + t.Parallel() + + stdout := &bytes.Buffer{} + app, err := NewApp(IO{In: strings.NewReader(""), Out: stdout, Err: &bytes.Buffer{}}) + if err != nil { + t.Fatalf("create app: %v", err) + } + + loaded := config.Loaded{ + Config: config.Config{ + Backend: "missing-backend", + Endpoint: "", + }, + Paths: config.Paths{ + GlobalPath: "/tmp/missing-global.yaml", + ProjectPath: "/tmp/missing-project.yaml", + }, + } + code, err := app.runDoctor(context.Background(), GlobalOptions{}, loaded, nil) + if err == nil { + t.Fatalf("expected doctor error") + } + if code != ExitBackend { + t.Fatalf("unexpected exit code: %d", code) + } + text := stdout.String() + if !strings.Contains(text, "What it tried:") { + t.Fatalf("doctor output missing tried section: %s", text) + } + if !strings.Contains(text, "What it found:") { + t.Fatalf("doctor output missing found section: %s", text) + } + if !strings.Contains(text, "Paste into .buildgraph.yaml:") { + t.Fatalf("doctor output missing config snippet section: %s", text) + } + if !strings.Contains(text, "Common fixes:") { + t.Fatalf("doctor output missing fixes section: %s", text) + } +} + +func TestRunTopJSONEnvelopeIncludesSchemaVersion(t *testing.T) { + t.Parallel() + + tracePath := writeTraceFixture(t) + stdout := &bytes.Buffer{} + app, err := NewApp(IO{In: strings.NewReader(""), Out: stdout, Err: &bytes.Buffer{}}) + if err != nil { + t.Fatalf("create app: %v", err) + } + + code, err := app.runTop(GlobalOptions{JSON: true}, []string{"--from", tracePath, "--limit", "2"}) + if err != nil { + t.Fatalf("run top json: %v", err) + } + if code != ExitOK { + t.Fatalf("unexpected exit code: %d", code) + } + + var env output.Envelope + if err := json.Unmarshal(stdout.Bytes(), &env); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if got, want := env.SchemaVersion, output.SchemaVersion; got != want { + t.Fatalf("unexpected schema version: got=%q want=%q", got, want) + } +} + +func writeTraceFixture(t *testing.T) string { + t.Helper() + + file, writer, err := trace.OpenFileWriter(filepath.Join(t.TempDir(), "trace.jsonl")) + if err != nil { + t.Fatalf("open trace writer: %v", err) + } + defer file.Close() + + t0 := time.Unix(0, 0).UTC() + t5 := time.Unix(0, int64(5*time.Millisecond)).UTC() + t20 := time.Unix(0, int64(20*time.Millisecond)).UTC() + + if err := writer.WriteRecord(trace.ProgressRecord("build", backend.BuildProgressEvent{ + VertexID: "a", + Message: "FROM alpine", + Started: &t0, + Completed: &t5, + })); err != nil { + t.Fatalf("write trace record a: %v", err) + } + if err := writer.WriteRecord(trace.ProgressRecord("build", backend.BuildProgressEvent{ + VertexID: "b", + Message: "RUN apk add curl", + Inputs: []string{"a"}, + Started: &t5, + Completed: &t20, + })); err != nil { + t.Fatalf("write trace record b: %v", err) + } + return file.Name() +} diff --git a/internal/output/human.go b/internal/output/human.go index 8111e88..fb3ffd5 100644 --- a/internal/output/human.go +++ b/internal/output/human.go @@ -61,16 +61,89 @@ func WriteBuild(w io.Writer, result backend.BuildResult) error { return nil } -func WriteDoctor(w io.Writer, checks map[string]string) error { - keys := make([]string, 0, len(checks)) - for key := range checks { +type DoctorReport struct { + Checks map[string]string `json:"checks"` + Attempts []backend.DetectAttempt `json:"attempts"` + Found backend.DetectResult `json:"found"` + ConfigSnippet string `json:"configSnippet"` + CommonFixes []string `json:"commonFixes"` +} + +func WriteDoctor(w io.Writer, report DoctorReport) error { + keys := make([]string, 0, len(report.Checks)) + for key := range report.Checks { keys = append(keys, key) } sort.Strings(keys) + + if _, err := fmt.Fprintln(w, "Checks:"); err != nil { + return err + } for _, key := range keys { - if _, err := fmt.Fprintf(w, "%s: %s\n", key, checks[key]); err != nil { + if _, err := fmt.Fprintf(w, "- %s: %s\n", key, report.Checks[key]); err != nil { + return err + } + } + + if _, err := fmt.Fprintln(w, "\nWhat it tried:"); err != nil { + return err + } + if len(report.Attempts) == 0 { + if _, err := fmt.Fprintln(w, "- (no backend detection attempts recorded)"); err != nil { return err } + } else { + for i, attempt := range report.Attempts { + line := fmt.Sprintf("%d. [%s] source=%s mode=%s endpoint=%s", i+1, strings.ToUpper(attempt.Status), attempt.Source, attempt.Mode, attempt.Endpoint) + if _, err := fmt.Fprintln(w, line); err != nil { + return err + } + if attempt.Details != "" { + if _, err := fmt.Fprintf(w, " details: %s\n", attempt.Details); err != nil { + return err + } + } + if attempt.Error != "" { + if _, err := fmt.Fprintf(w, " error: %s\n", attempt.Error); err != nil { + return err + } + } + } } + + if _, err := fmt.Fprintln(w, "\nWhat it found:"); err != nil { + return err + } + if report.Found.Available { + if _, err := fmt.Fprintf(w, "- backend: %s\n- mode: %s\n- endpoint: %s\n- details: %s\n", report.Found.Backend, report.Found.Mode, report.Found.Endpoint, report.Found.Details); err != nil { + return err + } + if source := report.Found.Metadata["resolutionSource"]; source != "" { + if _, err := fmt.Fprintf(w, "- resolution source: %s\n", source); err != nil { + return err + } + } + } else { + if _, err := fmt.Fprintf(w, "- backend detection failed: %s\n", report.Found.Details); err != nil { + return err + } + } + + if _, err := fmt.Fprintln(w, "\nPaste into .buildgraph.yaml:"); err != nil { + return err + } + if _, err := fmt.Fprintln(w, report.ConfigSnippet); err != nil { + return err + } + + if _, err := fmt.Fprintln(w, "Common fixes:"); err != nil { + return err + } + for _, fix := range report.CommonFixes { + if _, err := fmt.Fprintf(w, "- %s\n", fix); err != nil { + return err + } + } + return nil } diff --git a/internal/output/json.go b/internal/output/json.go index 739e7ed..9476338 100644 --- a/internal/output/json.go +++ b/internal/output/json.go @@ -7,6 +7,7 @@ import ( ) const APIVersion = "buildgraph.dev/v1" +const SchemaVersion = "1" type ErrorItem struct { Code string `json:"code"` @@ -14,12 +15,36 @@ type ErrorItem struct { } type Envelope struct { - APIVersion string `json:"apiVersion"` - Command string `json:"command"` - Timestamp time.Time `json:"timestamp"` - DurationMS int64 `json:"durationMs"` - Result any `json:"result"` - Errors []ErrorItem `json:"errors,omitempty"` + APIVersion string `json:"apiVersion"` + Command string `json:"command"` + SchemaVersion string `json:"schemaVersion"` + Timestamp time.Time `json:"timestamp"` + DurationMS int64 `json:"durationMs"` + Result any `json:"result"` + Errors []ErrorItem `json:"errors"` +} + +func NewEnvelope(command string, startedAt time.Time, result any, errors []ErrorItem) Envelope { + duration := time.Since(startedAt).Milliseconds() + if duration < 0 { + duration = 0 + } + return NewEnvelopeWithDuration(command, duration, result, errors) +} + +func NewEnvelopeWithDuration(command string, durationMS int64, result any, errors []ErrorItem) Envelope { + if errors == nil { + errors = []ErrorItem{} + } + return Envelope{ + APIVersion: APIVersion, + Command: command, + SchemaVersion: SchemaVersion, + Timestamp: time.Now().UTC(), + DurationMS: durationMS, + Result: result, + Errors: errors, + } } func WriteJSON(w io.Writer, v any) error { diff --git a/internal/output/json_test.go b/internal/output/json_test.go index 1a93f56..d7716cb 100644 --- a/internal/output/json_test.go +++ b/internal/output/json_test.go @@ -11,13 +11,15 @@ func TestWriteJSONEnvelope(t *testing.T) { t.Parallel() buf := &bytes.Buffer{} env := Envelope{ - APIVersion: APIVersion, - Command: "analyze", - Timestamp: time.Unix(0, 0).UTC(), - DurationMS: 12, + APIVersion: APIVersion, + Command: "analyze", + SchemaVersion: SchemaVersion, + Timestamp: time.Unix(0, 0).UTC(), + DurationMS: 12, Result: map[string]any{ "ok": true, }, + Errors: []ErrorItem{}, } if err := WriteJSON(buf, env); err != nil { t.Fatalf("write json: %v", err) @@ -26,4 +28,10 @@ func TestWriteJSONEnvelope(t *testing.T) { if !strings.Contains(text, `"apiVersion": "buildgraph.dev/v1"`) { t.Fatalf("apiVersion missing: %s", text) } + if !strings.Contains(text, `"schemaVersion": "1"`) { + t.Fatalf("schemaVersion missing: %s", text) + } + if !strings.Contains(text, `"errors": []`) { + t.Fatalf("errors array missing: %s", text) + } } diff --git a/internal/trace/graph.go b/internal/trace/graph.go new file mode 100644 index 0000000..5bd10c7 --- /dev/null +++ b/internal/trace/graph.go @@ -0,0 +1,368 @@ +package trace + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" +) + +var ErrNoVertexData = errors.New("trace does not contain vertex IDs; direct BuildKit progress is required for graph analysis") + +type Vertex struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Inputs []string `json:"inputs,omitempty"` + Started *time.Time `json:"started,omitempty"` + Completed *time.Time `json:"completed,omitempty"` + Cached bool `json:"cached"` + Error string `json:"error,omitempty"` + DurationMS int64 `json:"durationMs"` +} + +type Edge struct { + From string `json:"from"` + To string `json:"to"` +} + +type Graph struct { + Vertices []Vertex `json:"vertices"` + Edges []Edge `json:"edges"` +} + +type VertexSummary struct { + ID string `json:"id"` + Name string `json:"name"` + DurationMS int64 `json:"durationMs"` + Cached bool `json:"cached"` + Error string `json:"error,omitempty"` +} + +type CriticalPath struct { + DurationMS int64 `json:"durationMs"` + Vertices []VertexSummary `json:"vertices"` +} + +type TopResult struct { + VertexCount int `json:"vertexCount"` + Slowest []VertexSummary `json:"slowest"` + CriticalPath CriticalPath `json:"criticalPath"` + HasCycle bool `json:"hasCycle"` + CycleDetected string `json:"cycleDetected,omitempty"` +} + +func BuildGraph(records []Record) (Graph, error) { + vertices := map[string]*Vertex{} + + getVertex := func(id string) *Vertex { + if existing, ok := vertices[id]; ok { + return existing + } + created := &Vertex{ID: id} + vertices[id] = created + return created + } + + for _, rec := range records { + if rec.Kind != KindProgress || rec.Progress == nil { + continue + } + progress := rec.Progress + vertexID := strings.TrimSpace(progress.VertexID) + if vertexID == "" { + continue + } + + vertex := getVertex(vertexID) + if message := strings.TrimSpace(progress.Message); message != "" { + vertex.Name = message + } + if len(progress.Inputs) > 0 { + for _, input := range progress.Inputs { + trimmed := strings.TrimSpace(input) + if trimmed == "" || trimmed == vertexID { + continue + } + if !contains(vertex.Inputs, trimmed) { + vertex.Inputs = append(vertex.Inputs, trimmed) + } + } + } + if progress.Started != nil && (vertex.Started == nil || progress.Started.Before(*vertex.Started)) { + vertex.Started = cloneTime(progress.Started) + } + if progress.Completed != nil && (vertex.Completed == nil || progress.Completed.After(*vertex.Completed)) { + vertex.Completed = cloneTime(progress.Completed) + } + if progress.Cached { + vertex.Cached = true + } + if errText := strings.TrimSpace(progress.Error); errText != "" { + vertex.Error = errText + } + } + + if len(vertices) == 0 { + return Graph{}, ErrNoVertexData + } + + edgeKeys := map[string]struct{}{} + edges := make([]Edge, 0) + for _, vertex := range vertices { + sort.Strings(vertex.Inputs) + for _, input := range vertex.Inputs { + _ = getVertex(input) + key := input + "->" + vertex.ID + if _, ok := edgeKeys[key]; ok { + continue + } + edgeKeys[key] = struct{}{} + edges = append(edges, Edge{From: input, To: vertex.ID}) + } + if vertex.Started != nil && vertex.Completed != nil && vertex.Completed.After(*vertex.Started) { + vertex.DurationMS = vertex.Completed.Sub(*vertex.Started).Milliseconds() + } + } + + vertexIDs := make([]string, 0, len(vertices)) + for id := range vertices { + vertexIDs = append(vertexIDs, id) + } + sort.Strings(vertexIDs) + sort.Slice(edges, func(i, j int) bool { + if edges[i].From == edges[j].From { + return edges[i].To < edges[j].To + } + return edges[i].From < edges[j].From + }) + + result := Graph{ + Vertices: make([]Vertex, 0, len(vertexIDs)), + Edges: edges, + } + for _, id := range vertexIDs { + result.Vertices = append(result.Vertices, *vertices[id]) + } + return result, nil +} + +func AnalyzeTop(graph Graph, limit int) TopResult { + if limit <= 0 { + limit = 10 + } + + verticesByID := make(map[string]Vertex, len(graph.Vertices)) + for _, vertex := range graph.Vertices { + verticesByID[vertex.ID] = vertex + } + + summaries := make([]VertexSummary, 0, len(graph.Vertices)) + for _, vertex := range graph.Vertices { + summaries = append(summaries, summarizeVertex(vertex)) + } + sort.Slice(summaries, func(i, j int) bool { + if summaries[i].DurationMS == summaries[j].DurationMS { + if summaries[i].Name == summaries[j].Name { + return summaries[i].ID < summaries[j].ID + } + return summaries[i].Name < summaries[j].Name + } + return summaries[i].DurationMS > summaries[j].DurationMS + }) + if limit > len(summaries) { + limit = len(summaries) + } + + pathIDs, durationMS, hasCycle := longestPath(graph) + pathSummaries := make([]VertexSummary, 0, len(pathIDs)) + for _, id := range pathIDs { + vertex, ok := verticesByID[id] + if !ok { + continue + } + pathSummaries = append(pathSummaries, summarizeVertex(vertex)) + } + + result := TopResult{ + VertexCount: len(graph.Vertices), + Slowest: summaries[:limit], + CriticalPath: CriticalPath{DurationMS: durationMS, Vertices: pathSummaries}, + HasCycle: hasCycle, + } + if hasCycle { + result.CycleDetected = "trace graph contains cycles; critical path used fallback ordering" + } + return result +} + +func DOT(graph Graph) string { + var builder strings.Builder + builder.WriteString("digraph buildgraph {\n") + + for _, vertex := range graph.Vertices { + label := vertex.Name + if label == "" { + label = vertex.ID + } + if vertex.DurationMS > 0 { + label = fmt.Sprintf("%s\\n%d ms", label, vertex.DurationMS) + } + attrs := []string{fmt.Sprintf("label=\"%s\"", escapeDOT(label))} + if vertex.Cached { + attrs = append(attrs, "style=\"dashed\"") + } + if vertex.Error != "" { + attrs = append(attrs, "color=\"red\"") + } + builder.WriteString(fmt.Sprintf(" \"%s\" [%s];\n", escapeDOT(vertex.ID), strings.Join(attrs, ","))) + } + + for _, edge := range graph.Edges { + builder.WriteString(fmt.Sprintf(" \"%s\" -> \"%s\";\n", escapeDOT(edge.From), escapeDOT(edge.To))) + } + builder.WriteString("}\n") + return builder.String() +} + +func longestPath(graph Graph) ([]string, int64, bool) { + ids := make([]string, 0, len(graph.Vertices)) + durationByID := map[string]int64{} + inDegree := map[string]int{} + adjacent := map[string][]string{} + for _, vertex := range graph.Vertices { + ids = append(ids, vertex.ID) + durationByID[vertex.ID] = vertex.DurationMS + inDegree[vertex.ID] = 0 + } + sort.Strings(ids) + + for _, edge := range graph.Edges { + adjacent[edge.From] = append(adjacent[edge.From], edge.To) + inDegree[edge.To]++ + if _, ok := inDegree[edge.From]; !ok { + inDegree[edge.From] = 0 + ids = append(ids, edge.From) + durationByID[edge.From] = 0 + } + if _, ok := inDegree[edge.To]; !ok { + inDegree[edge.To] = 0 + ids = append(ids, edge.To) + durationByID[edge.To] = 0 + } + } + sort.Strings(ids) + + queue := make([]string, 0) + for _, id := range ids { + if inDegree[id] == 0 { + queue = append(queue, id) + } + } + sort.Strings(queue) + + topo := make([]string, 0, len(ids)) + for len(queue) > 0 { + sort.Strings(queue) + current := queue[0] + queue = queue[1:] + topo = append(topo, current) + for _, next := range adjacent[current] { + inDegree[next]-- + if inDegree[next] == 0 { + queue = append(queue, next) + } + } + } + + hasCycle := len(topo) != len(ids) + if hasCycle { + bestID := "" + var bestDuration int64 + for _, id := range ids { + if durationByID[id] > bestDuration || bestID == "" { + bestID = id + bestDuration = durationByID[id] + } + } + if bestID == "" { + return nil, 0, true + } + return []string{bestID}, bestDuration, true + } + + distance := map[string]int64{} + previous := map[string]string{} + for _, id := range topo { + distance[id] = durationByID[id] + } + for _, current := range topo { + for _, next := range adjacent[current] { + candidate := distance[current] + durationByID[next] + if candidate > distance[next] { + distance[next] = candidate + previous[next] = current + } + } + } + + end := "" + var maxDistance int64 + for _, id := range topo { + if distance[id] > maxDistance || end == "" { + end = id + maxDistance = distance[id] + } + } + if end == "" { + return nil, 0, false + } + + path := make([]string, 0) + for current := end; current != ""; current = previous[current] { + path = append(path, current) + if _, ok := previous[current]; !ok { + break + } + } + for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 { + path[i], path[j] = path[j], path[i] + } + return path, maxDistance, false +} + +func summarizeVertex(vertex Vertex) VertexSummary { + name := vertex.Name + if name == "" { + name = vertex.ID + } + return VertexSummary{ + ID: vertex.ID, + Name: name, + DurationMS: vertex.DurationMS, + Cached: vertex.Cached, + Error: vertex.Error, + } +} + +func cloneTime(value *time.Time) *time.Time { + if value == nil { + return nil + } + copied := value.UTC() + return &copied +} + +func contains(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} + +func escapeDOT(value string) string { + replacer := strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`) + return replacer.Replace(value) +} diff --git a/internal/trace/graph_test.go b/internal/trace/graph_test.go new file mode 100644 index 0000000..1a37879 --- /dev/null +++ b/internal/trace/graph_test.go @@ -0,0 +1,102 @@ +package trace + +import ( + "errors" + "testing" + "time" + + "github.com/Makepad-fr/buildgraph/internal/backend" +) + +func TestBuildGraphMergesVerticesAndComputesTop(t *testing.T) { + t.Parallel() + + t0 := time.Unix(0, 0).UTC() + t5 := time.Unix(0, int64(5*time.Millisecond)).UTC() + t6 := time.Unix(0, int64(6*time.Millisecond)).UTC() + t15 := time.Unix(0, int64(15*time.Millisecond)).UTC() + t20 := time.Unix(0, int64(20*time.Millisecond)).UTC() + + records := []Record{ + ProgressRecord("build", backend.BuildProgressEvent{ + VertexID: "a", + Message: "FROM alpine", + Started: &t0, + Completed: &t5, + }), + ProgressRecord("build", backend.BuildProgressEvent{ + VertexID: "b", + Message: "RUN apk add curl", + Inputs: []string{"a"}, + Started: &t5, + Completed: &t20, + }), + ProgressRecord("build", backend.BuildProgressEvent{ + VertexID: "c", + Message: "RUN echo hi", + Inputs: []string{"a"}, + Started: &t6, + Completed: &t15, + }), + ProgressRecord("build", backend.BuildProgressEvent{ + VertexID: "b", + Cached: true, + }), + } + + graph, err := BuildGraph(records) + if err != nil { + t.Fatalf("build graph: %v", err) + } + if got, want := len(graph.Vertices), 3; got != want { + t.Fatalf("unexpected vertex count: got=%d want=%d", got, want) + } + if got, want := len(graph.Edges), 2; got != want { + t.Fatalf("unexpected edge count: got=%d want=%d", got, want) + } + + var vertexB Vertex + for _, vertex := range graph.Vertices { + if vertex.ID == "b" { + vertexB = vertex + break + } + } + if got, want := vertexB.DurationMS, int64(15); got != want { + t.Fatalf("unexpected vertex b duration: got=%d want=%d", got, want) + } + if !vertexB.Cached { + t.Fatalf("expected merged cache=true for vertex b") + } + + top := AnalyzeTop(graph, 2) + if got, want := len(top.Slowest), 2; got != want { + t.Fatalf("unexpected slowest count: got=%d want=%d", got, want) + } + if got, want := top.Slowest[0].ID, "b"; got != want { + t.Fatalf("unexpected slowest[0]: got=%q want=%q", got, want) + } + if got, want := top.CriticalPath.DurationMS, int64(20); got != want { + t.Fatalf("unexpected critical path duration: got=%d want=%d", got, want) + } + if got, want := len(top.CriticalPath.Vertices), 2; got != want { + t.Fatalf("unexpected critical path length: got=%d want=%d", got, want) + } + if got, want := top.CriticalPath.Vertices[0].ID, "a"; got != want { + t.Fatalf("unexpected critical path first vertex: got=%q want=%q", got, want) + } + if got, want := top.CriticalPath.Vertices[1].ID, "b"; got != want { + t.Fatalf("unexpected critical path second vertex: got=%q want=%q", got, want) + } +} + +func TestBuildGraphRequiresVertexIDs(t *testing.T) { + t.Parallel() + + _, err := BuildGraph([]Record{ + ProgressRecord("build", backend.BuildProgressEvent{Message: "plain log"}), + }) + if !errors.Is(err, ErrNoVertexData) { + t.Fatalf("expected ErrNoVertexData, got %v", err) + } +} diff --git a/internal/trace/trace.go b/internal/trace/trace.go new file mode 100644 index 0000000..a80a64f --- /dev/null +++ b/internal/trace/trace.go @@ -0,0 +1,143 @@ +package trace + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/Makepad-fr/buildgraph/internal/backend" +) + +const SchemaVersion = "1" + +const ( + KindProgress = "progress" + KindResult = "result" + KindError = "error" +) + +type ErrorRecord struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type Record struct { + SchemaVersion string `json:"schemaVersion"` + Kind string `json:"kind"` + Command string `json:"command"` + Timestamp time.Time `json:"timestamp"` + Progress *backend.BuildProgressEvent `json:"progress,omitempty"` + Result *backend.BuildResult `json:"result,omitempty"` + Error *ErrorRecord `json:"error,omitempty"` +} + +type Writer struct { + mu sync.Mutex + enc *json.Encoder +} + +func NewWriter(w io.Writer) *Writer { + return &Writer{ + enc: json.NewEncoder(w), + } +} + +func (w *Writer) WriteRecord(rec Record) error { + if w == nil || w.enc == nil { + return fmt.Errorf("trace writer is not initialized") + } + rec = normalizedRecord(rec) + w.mu.Lock() + defer w.mu.Unlock() + return w.enc.Encode(rec) +} + +func OpenFileWriter(path string) (*os.File, *Writer, error) { + if path == "" { + return nil, nil, fmt.Errorf("trace path is required") + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, nil, fmt.Errorf("create trace parent dir: %w", err) + } + file, err := os.Create(path) + if err != nil { + return nil, nil, fmt.Errorf("create trace file: %w", err) + } + return file, NewWriter(file), nil +} + +func ReadRecords(r io.Reader) ([]Record, error) { + dec := json.NewDecoder(bufio.NewReader(r)) + records := make([]Record, 0) + for { + var rec Record + err := dec.Decode(&rec) + if errors.Is(err, io.EOF) { + return records, nil + } + if err != nil { + return nil, fmt.Errorf("decode trace record: %w", err) + } + records = append(records, normalizedRecord(rec)) + } +} + +func LoadFile(path string) ([]Record, error) { + if path == "" { + return nil, fmt.Errorf("trace path is required") + } + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open trace file: %w", err) + } + defer file.Close() + return ReadRecords(file) +} + +func ProgressRecord(command string, event backend.BuildProgressEvent) Record { + cloned := event + return Record{ + Kind: KindProgress, + Command: command, + Timestamp: time.Now().UTC(), + Progress: &cloned, + } +} + +func ResultRecord(command string, result backend.BuildResult) Record { + cloned := result + return Record{ + Kind: KindResult, + Command: command, + Timestamp: time.Now().UTC(), + Result: &cloned, + } +} + +func FailureRecord(command, code, message string) Record { + return Record{ + Kind: KindError, + Command: command, + Timestamp: time.Now().UTC(), + Error: &ErrorRecord{ + Code: code, + Message: message, + }, + } +} + +func normalizedRecord(rec Record) Record { + if rec.SchemaVersion == "" { + rec.SchemaVersion = SchemaVersion + } + if rec.Timestamp.IsZero() { + rec.Timestamp = time.Now().UTC() + } + return rec +} diff --git a/internal/trace/trace_test.go b/internal/trace/trace_test.go new file mode 100644 index 0000000..ad6317d --- /dev/null +++ b/internal/trace/trace_test.go @@ -0,0 +1,54 @@ +package trace + +import ( + "bytes" + "testing" + "time" + + "github.com/Makepad-fr/buildgraph/internal/backend" +) + +func TestTraceRoundTripNDJSON(t *testing.T) { + t.Parallel() + + buf := &bytes.Buffer{} + writer := NewWriter(buf) + if err := writer.WriteRecord(ProgressRecord("build", backend.BuildProgressEvent{ + Timestamp: time.Unix(1, 0).UTC(), + Phase: "build", + Message: "step", + VertexID: "v1", + Status: "running", + })); err != nil { + t.Fatalf("write progress: %v", err) + } + if err := writer.WriteRecord(ResultRecord("build", backend.BuildResult{Digest: "sha256:test"})); err != nil { + t.Fatalf("write result: %v", err) + } + if err := writer.WriteRecord(FailureRecord("build", "build_failed", "boom")); err != nil { + t.Fatalf("write error: %v", err) + } + + records, err := ReadRecords(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatalf("read records: %v", err) + } + if got, want := len(records), 3; got != want { + t.Fatalf("unexpected record count: got=%d want=%d", got, want) + } + if got, want := records[0].SchemaVersion, SchemaVersion; got != want { + t.Fatalf("unexpected schema version: got=%q want=%q", got, want) + } + if got, want := records[0].Kind, KindProgress; got != want { + t.Fatalf("unexpected first kind: got=%q want=%q", got, want) + } + if got, want := records[1].Kind, KindResult; got != want { + t.Fatalf("unexpected second kind: got=%q want=%q", got, want) + } + if got, want := records[2].Kind, KindError; got != want { + t.Fatalf("unexpected third kind: got=%q want=%q", got, want) + } + if records[2].Error == nil || records[2].Error.Message != "boom" { + t.Fatalf("unexpected error payload: %+v", records[2].Error) + } +} diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..396ed50 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,24 @@ +site_name: Buildgraph Docs +site_description: Documentation for Buildgraph rules and CLI diagnostics. +site_url: https://docs.buildgraph.dev +docs_dir: docs +site_dir: site +exclude_docs: | + README.md +theme: + name: mkdocs +nav: + - Home: index.md + - Rules: + - Overview: rules/index.md + - BG_PERF_APT_SPLIT: rules/BG_PERF_APT_SPLIT.md + - BG_PERF_TOO_MANY_RUN: rules/BG_PERF_TOO_MANY_RUN.md + - BG_CACHE_COPY_ALL_EARLY: rules/BG_CACHE_COPY_ALL_EARLY.md + - BG_CACHE_ARG_LATE: rules/BG_CACHE_ARG_LATE.md + - BG_REPRO_FROM_MUTABLE: rules/BG_REPRO_FROM_MUTABLE.md + - BG_REPRO_APT_UNPINNED: rules/BG_REPRO_APT_UNPINNED.md + - BG_SEC_ROOT_USER: rules/BG_SEC_ROOT_USER.md + - BG_SEC_CURL_PIPE_SH: rules/BG_SEC_CURL_PIPE_SH.md + - BG_SEC_PLAIN_SECRET_ENV: rules/BG_SEC_PLAIN_SECRET_ENV.md + - BG_POL_MISSING_SOURCE_LABEL: rules/BG_POL_MISSING_SOURCE_LABEL.md + - BG_POL_MISSING_HEALTHCHECK: rules/BG_POL_MISSING_HEALTHCHECK.md