diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..829380c4 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,241 @@ +# syntax=docker/dockerfile:1.7 +# tableauio/loader devcontainer +# +# Single-stage, multi-arch (amd64 + arm64) image bringing the full +# C++/Go/.NET toolchain plus protobuf at the exact versions CI uses. +# +# All version pins are read from ./versions.env (the single source of +# truth shared with make.py and the CI workflows). To bump Go / buf / +# protobuf / .NET / vcpkg-baseline, edit that file — not this one. +# +# Build context is .devcontainer/ (the directory containing this file), +# so `COPY versions.env ...` resolves directly. + +FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 + +# --------------------------------------------------------------------------- +# Pull pinned versions from the shared file. We `COPY` it into a build-stage +# location and `source` it in every RUN that needs the values; ARG/ENV alone +# don't survive across RUN boundaries in BuildKit, so we re-source per layer. +# --------------------------------------------------------------------------- +COPY versions.env /opt/versions.env + +# --------------------------------------------------------------------------- +# Architecture detection. BuildKit auto-populates TARGETARCH; we resolve it +# into per-arch download-name fragments (Go's tarball, buf's release asset, +# vcpkg triplet) and persist them to /opt/buildargs.env so later RUN layers +# can `source` them — Dockerfile ARGs don't survive across RUN boundaries. +# --------------------------------------------------------------------------- +ARG TARGETARCH +RUN < /opt/buildargs.env +EOF + +# --------------------------------------------------------------------------- +# Go — official tarball into /usr/local/go. +# +# PATH is set via ENV (not /etc/profile.d/) so non-interactive shells like +# the postCreateCommand and downstream RUNs see Go without sourcing profile. +# /home/vscode/go/bin lands `go install`-placed binaries on PATH automatically. +# --------------------------------------------------------------------------- +RUN <&2 + exit 1 + ;; +esac + +# Surgical override: when LOADER_PROTOBUF_VERSION is set, use it for the +# protobuf pin only — keep the variant's vcpkg baseline. Mostly useful for +# bisecting protobuf releases against a known-good vcpkg snapshot. +if [ -n "${_ARG_PROTOBUF_VERSION:-}" ]; then + PROTOBUF_VERSION="${_ARG_PROTOBUF_VERSION}" +fi + +# 1. Bring up vcpkg pinned to the baseline commit. +git clone https://github.com/microsoft/vcpkg.git "${VCPKG_ROOT}" +git -C "${VCPKG_ROOT}" checkout "${VCPKG_BASELINE_COMMIT}" +"${VCPKG_ROOT}/bootstrap-vcpkg.sh" -disableMetrics + +# Make sure the binary-cache dir exists. The RUN's --mount=type=cache +# creates it, but vcpkg can be picky about owner/perms on first use. +mkdir -p "${VCPKG_DEFAULT_BINARY_CACHE}" + +# 2. Render a minimal manifest with builtin-baseline + the protobuf override. +mkdir -p /opt/vcpkg-manifest +cat > /opt/vcpkg-manifest/vcpkg.json </dev/null | head -n1) +case "$(basename "${INFO_FILE:-/missing}" 2>/dev/null)" in + protobuf_${PROTOBUF_VERSION}*) + ;; + *) + echo "ERROR: installed protobuf does not match requested version ${PROTOBUF_VERSION}." + echo " vcpkg installed-file marker: ${INFO_FILE:-}" + echo " Bump VCPKG_BASELINE_COMMIT in .devcontainer/versions.env" + echo " (and make.py + testing-cpp.yml will pick it up automatically)" + echo " to a commit that knows about the requested version." + exit 1 + ;; +esac + +# 5. Stable symlinks so ENV CMAKE_PREFIX_PATH (last layer) doesn't have to +# care about the underlying triplet. +ln -s /opt/vcpkg-manifest/vcpkg_installed/${VCPKG_TRIPLET} /opt/vcpkg/active +ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc +EOF + +# --------------------------------------------------------------------------- +# .NET SDK — apt-based install from the official Microsoft repository. +# Version is read from /opt/versions.env. +# apt-get clean + rm /var/lib/apt/lists at the end keeps the layer small. +# --------------------------------------------------------------------------- +RUN <` for any language. + +## Pin a different protobuf version + +```sh +LOADER_PROTOBUF_VERSION=3.21.12 code . +``` + +Then **Rebuild Container**. Only the vcpkg layer rebuilds. + +## Host-OS caveats + +- **Windows.** WSL2 backend required. Check the workspace out under WSL2 (`\\wsl.localhost\Ubuntu\home\\loader`) — not `/mnt/c/...` — for good bind-mount performance. +- **Apple Silicon.** Docker builds the image natively as arm64. Confirm with `docker info | grep Architecture`. +- **Linux (native Docker Engine).** No special configuration. + +## Architecture + +Single-stage Dockerfile on `mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`: + +1. Architecture detection (`TARGETARCH` → Go arch, buf arch, vcpkg triplet) +2. Go — version from `versions.env` +3. buf — version from `versions.env` +4. vcpkg pinned to `VCPKG_BASELINE_COMMIT`, protobuf installed via manifest mode (asserts version) +5. .NET SDK — version from `versions.env` +6. Node.js LTS — version from `versions.env` +7. `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` so `find_package(Protobuf CONFIG)` resolves automatically + +Build context is `.devcontainer/`, so `COPY versions.env …` resolves directly. + +## `versions.env` format + +- One `KEY=VALUE` per line. +- No quotes, no spaces around `=`, no inline comments. +- Comments at column 0 with `#`. Blank lines ignored. No shell expansion. + +Consumers: Dockerfile (sourced as a shell file), [`make.py`](../make.py) (`Versions.load()`), and `.github/actions/load-versions` (exports to `$GITHUB_ENV`). + +Use `python3 make.py env` for a JSON dump of resolved values. + +## Troubleshooting + +### Stale-codegen errors after `buf generate` + +Symptoms: +- C++: `fatal error: google/protobuf/generated_message_table_driven.h: No such file or directory` +- C#: hundreds of `error CS0101: The namespace already contains a definition for ...` + +The host workspace has gitignored `*.pb.*` from a previous protobuf version that `git pull` didn't remove. Wipe with `python3 make.py clean --lang cpp` (or `--lang csharp`), then rerun the test. + +## Falling back + +No Docker? Use [`make.py`](../make.py) directly: `python3 make.py setup --lang all` then `python3 make.py test --lang `. Works on macOS / Linux / Windows native. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..40c609e0 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,67 @@ +{ + // The loader's only devcontainer. Multi-arch: builds linux/amd64 on + // x64 hosts, linux/arm64 on Apple Silicon and Windows-on-ARM. Use + // this on every host that can run Docker Desktop or Docker Engine. + // Native users (no Docker) bootstrap via `python3 make.py setup` at + // the repo root. + // + // See ./versions.env for all toolchain pins. + "name": "tableauio/loader", + // Build args wire host env to Dockerfile ARGs: + // LOADER_DEFAULT_VARIANT on the host -> DEFAULT_VARIANT inside. + // Switches both protobuf AND vcpkg baseline atomically; values come + // from versions.env's MODERN_*/LEGACY_V3_* rows. + // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. + // Surgical: overrides ONLY the protobuf version, vcpkg baseline + // stays at the variant's pin. For a clean variant switch, prefer + // LOADER_DEFAULT_VARIANT. + // Both default empty (versions.env's DEFAULT_VARIANT row wins). Example: + // LOADER_DEFAULT_VARIANT=legacy-v3 code . # then Reopen in Container. + // + // "context" defaults to the directory holding this devcontainer.json + // (.devcontainer/), so the Dockerfile's `COPY versions.env ...` + // resolves directly. + "build": { + "dockerfile": "Dockerfile", + "args": { + "DEFAULT_VARIANT": "${localEnv:LOADER_DEFAULT_VARIANT:}", + "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:}" + } + }, + // Persist the Go module cache across container rebuilds. Workspace itself + // uses VS Code's default bind-mount so edits sync to the host. + "mounts": [ + "source=loader-go-mod,target=/home/vscode/go,type=volume" + ], + "remoteUser": "vscode", + "workspaceFolder": "/workspaces/loader", + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-dotnettools.csharp", + "bufbuild.vscode-buf", + "DrBlury.protobuf-vsc" + ], + "settings": { + // Don't auto-install gopls and friends on first open — let the user + // do it explicitly from the Go extension's command palette. + "go.toolsManagement.autoUpdate": false, + // Don't auto-cmake-configure on workspace open; we run cmake manually + // per the existing README recipes. + "cmake.configureOnOpen": false + } + } + }, + // Ready banner so the developer knows the container is healthy. + // Script is baked into the image at build time (see Dockerfile). + "postCreateCommand": "/usr/local/bin/loader-devcontainer-banner", + // The loader-go-mod named volume is mounted at /home/vscode/go (see "mounts" + // above). Docker creates named volumes root-owned by default, masking any + // ownership set in the Dockerfile. Re-chown on every start so `go install` + // (e.g. gopls auto-install from the Go extension) can write into GOPATH. + // Idempotent and ~50ms on an already-correct tree. + "postStartCommand": "sudo chown -R vscode:vscode /home/vscode/go" +} \ No newline at end of file diff --git a/.devcontainer/postcreate-banner.sh b/.devcontainer/postcreate-banner.sh new file mode 100644 index 00000000..ec436498 --- /dev/null +++ b/.devcontainer/postcreate-banner.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Post-create banner for the Linux devcontainer. +# Pure echo — no installs, no version-pinning at runtime, no surprises. +# Four-line summary that prints when the container becomes ready, so the +# developer can confirm at a glance which toolchain versions landed. +set -e +printf 'tableauio/loader devcontainer ready (linux).\n' +printf ' go: %s\n' "$(go version | cut -d' ' -f3)" +printf ' buf: %s\n' "$(buf --version 2>&1)" +printf ' protoc: %s\n' "$(protoc --version)" +printf ' dotnet: %s\n' "$(dotnet --version)" diff --git a/.devcontainer/versions.env b/.devcontainer/versions.env new file mode 100644 index 00000000..893983e5 --- /dev/null +++ b/.devcontainer/versions.env @@ -0,0 +1,66 @@ +# tableauio/loader — pinned toolchain versions. +# +# Single source of truth consumed by: +# - .devcontainer/Dockerfile (sourced as a shell file) +# - make.py (parsed via Versions.load()) +# - .github/workflows/*.yml (read by the load-versions composite action into $GITHUB_ENV) +# +# Format rules so every consumer can parse this file with a single regex: +# - One KEY=VALUE per line, no quotes, no spaces around `=`. +# - Comments start with `#` at column 0 only (not after a value). +# - Blank lines are ignored. +# - Values are bare strings — no shell expansion, no $VAR references. + +# Go SDK shipped in the devcontainer. Must be ≥ the `go` directive in go.mod; +# CI workflows resolve their Go version from go.mod via setup-go's +# go-version-file, so devcontainer is the only place Go is hard-pinned. +GO_VERSION=1.24.0 + +# buf CLI release. Pinned in the devcontainer, make.py, and all three +# testing-*.yml workflows. Bump everywhere together. +BUF_VERSION=1.67.0 + +# --------------------------------------------------------------------------- +# protobuf / vcpkg matrix. +# +# Each variant pins a (protobuf, vcpkg-snapshot) pair where the snapshot's +# baseline.json knows about that protobuf version (so abseil / utf8-range / +# re2 transitive deps come from one self-consistent vcpkg cut). DEFAULT_VARIANT +# selects which row the devcontainer image and `make.py setup` install. CI's +# .github/workflows/testing-cpp.yml mirrors these labels in its matrix. +# +# Override at devcontainer build time: set DEFAULT_VARIANT in the host env +# before "Rebuild Container". +# +# Override per-invocation in `make.py`: pass --protobuf-version / --vcpkg-baseline. +# +# Bumping a row: pick the new commit pointed to by a recent quarterly tag at +# https://github.com/microsoft/vcpkg/tags (e.g. `git rev-parse 2026.04.27^{commit}`). +# Use the *commit* SHA, not the tag-object SHA: GitHub's /commit/ view +# 404s on tag objects, and the resolved manifest baseline is the commit anyway. +# Verify the chosen commit's versions/baseline.json has +# default.protobuf.baseline matching the requested protobuf version. +# --------------------------------------------------------------------------- + +# Active variant — Dockerfile, load-versions action, and make.py read this +# to resolve PROTOBUF_VERSION / VCPKG_BASELINE_COMMIT to the matching row. +DEFAULT_VARIANT=modern + +# Modern variant: protobuf 6.x line. Currently pins the tip of the +# `2026.04.27` quarterly vcpkg release. +MODERN_PROTOBUF_VERSION=6.33.4 +MODERN_VCPKG_BASELINE_COMMIT=56bb2411609227288b70117ead2c47585ba07713 + +# Legacy v3 variant: protobuf 3.x ABI smoke test. Snapshot from 2023-01-15. +# Linux-only in CI — the windows port pulls MSYS2 distfiles that have since +# been pruned from repo.msys2.org (see testing-cpp.yml's matrix exclude). +LEGACY_V3_PROTOBUF_VERSION=3.21.12 +LEGACY_V3_VCPKG_BASELINE_COMMIT=6245ce44a03f04d19be125ab1bbab578d0933e85 + +# .NET SDK major.minor. apt installs `dotnet-sdk-${DOTNET_VERSION}` on Linux. +# CI uses `${DOTNET_VERSION}.x` with actions/setup-dotnet. +DOTNET_VERSION=8.0 + +# CMake version installed by `make.py setup` on Windows (the devcontainer +# base image already ships a recent cmake; macOS/Linux use system cmake). +CMAKE_VERSION=3.31.8 diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml new file mode 100644 index 00000000..ff6d0ef1 --- /dev/null +++ b/.github/actions/load-versions/action.yml @@ -0,0 +1,54 @@ +name: Load pinned versions +description: > + Reads .devcontainer/versions.env (the single source of truth shared + with the devcontainer and make.py) and exports each KEY=VALUE pair to + GITHUB_ENV so subsequent steps can reference them as env-context values + (e.g. env.BUF_VERSION). + + Also resolves the active (DEFAULT_VARIANT) row's MODERN_/LEGACY_V3_-prefixed + protobuf+vcpkg pins back to the unprefixed PROTOBUF_VERSION / + VCPKG_BASELINE_COMMIT names so existing workflow references keep working. + + Format rules of versions.env (kept simple so every consumer can parse it + with a builtin): + - One KEY=VALUE per line, no quotes, no spaces around `=`. + - Comments start with `#` at column 0. + - Blank lines are ignored. + +runs: + using: composite + steps: + - name: Export versions.env to $GITHUB_ENV + shell: bash + run: | + set -eu + file=.devcontainer/versions.env + if [ ! -f "$file" ]; then + echo "::error::Missing $file; cannot resolve pinned tool versions." + exit 1 + fi + # Pass 1: every KEY=VALUE → $GITHUB_ENV verbatim. + declare -A kv + while IFS='=' read -r k v; do + [ -z "$k" ] && continue + case "$k" in \#*) continue ;; esac + printf '%s=%s\n' "$k" "$v" >> "$GITHUB_ENV" + kv[$k]=$v + done < "$file" + + # Pass 2: resolve the active variant's protobuf + vcpkg pins back to + # the unprefixed names so workflows can keep referencing + # ${{ env.PROTOBUF_VERSION }} / ${{ env.VCPKG_BASELINE_COMMIT }}. + variant="${kv[DEFAULT_VARIANT]:-modern}" + prefix=$(printf '%s' "$variant" | tr '[:lower:]-' '[:upper:]_') + protobuf_key="${prefix}_PROTOBUF_VERSION" + vcpkg_key="${prefix}_VCPKG_BASELINE_COMMIT" + if [ -z "${kv[$protobuf_key]:-}" ] || [ -z "${kv[$vcpkg_key]:-}" ]; then + echo "::error::variant '$variant' missing $protobuf_key and/or $vcpkg_key in $file" + exit 1 + fi + printf 'PROTOBUF_VERSION=%s\n' "${kv[$protobuf_key]}" >> "$GITHUB_ENV" + printf 'VCPKG_BASELINE_COMMIT=%s\n' "${kv[$vcpkg_key]}" >> "$GITHUB_ENV" + # Mirror VCPKG_BASELINE_COMMIT under VCPKG_COMMIT for backward + # compat with testing-cpp.yml's pre-existing variable name. + printf 'VCPKG_COMMIT=%s\n' "${kv[$vcpkg_key]}" >> "$GITHUB_ENV" diff --git a/.github/workflows/devcontainer-smoke.yml b/.github/workflows/devcontainer-smoke.yml new file mode 100644 index 00000000..947c38d7 --- /dev/null +++ b/.github/workflows/devcontainer-smoke.yml @@ -0,0 +1,114 @@ +name: Devcontainer Smoke + +# Builds the devcontainer image and runs a minimal smoke test inside it. +# Triggered only when files that affect the image change, so we don't +# burn CI minutes on unrelated PRs. +# +# What this catches that testing-cpp.yml does not: +# 1. arm64 coverage. The Dockerfile claims linux/amd64 + linux/arm64; +# testing-cpp.yml only exercises amd64. This workflow builds the +# image natively on ubuntu-24.04-arm runners. +# 2. versions.env regressions. A typo in the shared file (stray space, +# quote, etc.) silently breaks the devcontainer build for anyone +# running `Reopen in Container`. The testing-*.yml workflows +# consume versions.env too, but only the keys they need; only this +# workflow exercises the full file in a real Dockerfile build. +# +# What this DOES NOT do: +# - Run the full C++ / C# test matrices. Those are testing-{cpp,csharp}.yml, +# which use lukka/run-vcpkg directly on the runner (much faster than +# building the devcontainer image first). The container is for +# local-dev parity, not the primary CI test path. + +on: + pull_request: + paths: + - ".devcontainer/**" + - ".github/workflows/devcontainer-smoke.yml" + push: + branches: [master, main] + paths: + - ".devcontainer/**" + workflow_dispatch: + +permissions: + contents: read + # actions: write is required for `cache-to: type=gha` to push BuildKit + # cache entries to the per-repo Actions cache (~10 GB free quota). + actions: write + +jobs: + build-and-smoke: + strategy: + fail-fast: false + matrix: + # amd64 is partially covered by testing-cpp.yml's ubuntu-latest + # job (vcpkg+protobuf, x64-linux). arm64 is the genuinely-new + # coverage this workflow adds: GitHub-hosted ubuntu-24.04-arm + # runners build the image natively, no QEMU emulation. + include: + - runner: ubuntu-latest + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + name: build devcontainer + smoke (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build devcontainer image + # Use docker/build-push-action so we can point BuildKit at GitHub's + # Actions cache (10 GB free per repo). Cold first run is unchanged + # (~25 min — vcpkg compiles protobuf from source). Subsequent runs + # restore every layer from cache and complete in ~3-5 min, matching + # what testing-cpp.yml achieves via its actions/cache step on + # ${VCPKG_INSTALLED_DIR}. + # + # mode=max writes every intermediate layer to cache, not just the + # final image. That trades cache size (still under the 10 GB ceiling + # for this image) for the ability to skip vcpkg's compile layer on + # any later run, even when an unrelated layer above it changed. + uses: docker/build-push-action@v6 + with: + context: .devcontainer + file: .devcontainer/Dockerfile + tags: loader-devcontainer:smoke + load: true + cache-from: type=gha,scope=devcontainer-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=devcontainer-${{ matrix.arch }} + + - name: Smoke — version banner + run: | + # Confirms the image runs at all and that all five toolchain + # binaries (go, buf, protoc, dotnet, node) resolve. + docker run --rm loader-devcontainer:smoke \ + /usr/local/bin/loader-devcontainer-banner + + - name: Smoke — buf generate (Go) + run: | + # Bind-mount the checkout into the container's workspace dir + # and regenerate Go protoconf via make.py. Catches protoc/buf + # integration regressions that wouldn't show up in the banner. + docker run --rm \ + -v "${{ github.workspace }}:/workspaces/loader" \ + -w /workspaces/loader \ + loader-devcontainer:smoke \ + python3 make.py generate --lang go + + - name: Smoke — go vet (plugin packages only) + run: | + # Vet the plugin sources and shared internal packages via the + # make.py --smoke shortcut. Skips ./test/... and ./internal/index + # — those depend on freshly generated *.pb.go that we don't + # produce in this smoke job. + docker run --rm \ + -v "${{ github.workspace }}:/workspaces/loader" \ + -w /workspaces/loader \ + loader-devcontainer:smoke \ + python3 make.py test --lang go --smoke diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 80151d9b..afc1ab3c 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -15,24 +15,71 @@ permissions: jobs: test: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - protobuf-version: ["32.0", "3.19.3"] + config: + # vcpkg-commit pins the `builtin-baseline` for each protobuf + # version. We use the baseline (not `overrides`) so that abseil / + # utf8-range / re2 are pulled from the same self-consistent vcpkg + # snapshot as protobuf — `overrides` alone would let those + # transitive deps drift forward and break ABI. + # + # The labels and SHAs below mirror .devcontainer/versions.env's + # MODERN_*/LEGACY_V3_* entries — keep them in sync when bumping. + # GitHub Actions matrices are evaluated at workflow parse time, + # so we can't read them from versions.env via the load-versions + # composite action; the duplication is intentional. + # + # CI passes the SHA explicitly via --vcpkg-baseline so the run is + # fully reproducible and independent of vcpkg's git history (which + # `lukka/run-vcpkg` checks out shallowly). For ad-hoc local runs + # `make.py` can auto-resolve the SHA from $VCPKG_ROOT instead — see + # `_resolve_vcpkg_baseline_for_protobuf`. + # + # To pin a new protobuf version, run on a full vcpkg checkout: + # git -C $VCPKG_ROOT log -S '"X.Y.Z"' -- versions/baseline.json + # then pick the newest commit whose baseline.json has + # default.protobuf.baseline == "X.Y.Z". + - label: modern + protobuf-version: "6.33.4" + vcpkg-commit: "56bb2411609227288b70117ead2c47585ba07713" + - label: legacy-v3 + protobuf-version: "3.21.12" + vcpkg-commit: "6245ce44a03f04d19be125ab1bbab578d0933e85" include: - os: ubuntu-latest - init_script: bash init.sh + triplet: x64-linux - os: windows-latest - init_script: cmd /c init.bat + triplet: x64-windows-static + exclude: + # The 2023-01-15 vcpkg snapshot pinned by legacy-v3 references + # MSYS2 distfiles that have since been pruned from repo.msys2.org; + # the protobuf:x64-windows port pulls them in via vcpkg_acquire_msys + # → vcpkg_fixup_pkgconfig and fails the download. Linux uses the + # system pkg-config so it isn't affected. Keep legacy-v3 as a + # Linux-only smoke test for the protobuf-3 ABI; modern still + # exercises the Windows build path. + - os: windows-latest + config: + label: legacy-v3 + protobuf-version: "3.21.12" + vcpkg-commit: "6245ce44a03f04d19be125ab1bbab578d0933e85" - name: test (${{ matrix.os }}, protobuf ${{ matrix.protobuf-version }}) + name: test (${{ matrix.os }}, ${{ matrix.config.label }}) runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 45 + + env: + VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed + VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} steps: - name: Checkout Code uses: actions/checkout@v6 - with: - submodules: true + + - name: Read pinned versions + uses: ./.github/actions/load-versions - name: Install Go uses: actions/setup-go@v6 @@ -41,7 +88,7 @@ jobs: cache-dependency-path: go.sum cache: true - - name: Install dependencies (Ubuntu) + - name: Install Ninja (Ubuntu) if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y ninja-build @@ -49,54 +96,74 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 - - name: Cache protobuf install - id: cache-protobuf + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Render vcpkg.json + # `lukka/run-vcpkg` runs `vcpkg install` before `make.py test`, so + # the manifest must exist beforehand. We render the same shape + # `make.py` would (baseline-only, no `overrides`) keyed on the same + # SHA, so the cache key and the eventual build see one manifest. + working-directory: test/cpp-tableau-loader + shell: bash + run: | + cat > vcpkg.json <> "$GITHUB_PATH" + + - name: Add vcpkg-installed protoc to PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: Add-Content -Path $env:GITHUB_PATH -Value "$env:VCPKG_INSTALLED_DIR\${{ matrix.triplet }}\tools\protobuf" - name: Install Buf uses: bufbuild/buf-action@v1 with: - version: 1.67.0 + version: ${{ env.BUF_VERSION }} setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Generate protoconf - working-directory: test/cpp-tableau-loader - run: buf generate .. - - - name: CMake Configure - working-directory: test/cpp-tableau-loader - run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=17 - - - name: CMake Build - working-directory: test/cpp-tableau-loader - run: cmake --build build --parallel - - - name: Run tests - working-directory: test/cpp-tableau-loader - run: ctest --test-dir build --output-on-failure + - name: Test + # Pass --vcpkg-baseline explicitly so make.py reuses the SHA we + # already rendered into vcpkg.json above, instead of auto-resolving + # from the vcpkg checkout (which is shallow on the CI runner). + run: > + python3 make.py test --lang cpp + --protobuf-version ${{ matrix.config.protobuf-version }} + --vcpkg-baseline ${{ matrix.config.vcpkg-commit }} + --triplet ${{ matrix.triplet }} + --no-clean + --no-vcpkg-install diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index 06b19f57..eef6e880 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -15,11 +15,11 @@ permissions: jobs: test: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - protobuf-version: ["32.0", "3.19.3"] - name: test (${{ matrix.os }}, protobuf ${{ matrix.protobuf-version }}) + name: test (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 10 @@ -29,6 +29,9 @@ jobs: with: submodules: recursive + - name: Read pinned versions + uses: ./.github/actions/load-versions + - name: Install Go uses: actions/setup-go@v6 with: @@ -39,33 +42,27 @@ jobs: - name: Install .NET SDK uses: actions/setup-dotnet@v4 with: - dotnet-version: "8.0.x" + dotnet-version: "${{ env.DOTNET_VERSION }}.x" - name: Install Protoc - if: "!startsWith(matrix.protobuf-version, '3.')" + # C# consumes only generated `.cs` files; libprotobuf is irrelevant, + # so a single modern protoc release is enough (no legacy-v3 matrix). uses: arduino/setup-protoc@v3 with: - version: ${{ matrix.protobuf-version }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install Protoc (legacy) - if: startsWith(matrix.protobuf-version, '3.') - uses: arduino/setup-protoc@v1 - with: - version: ${{ matrix.protobuf-version }} + version: "33.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install Buf uses: bufbuild/buf-action@v1 with: - version: 1.67.0 + version: ${{ env.BUF_VERSION }} setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Generate protoconf - working-directory: test/csharp-tableau-loader - run: buf generate .. + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' - name: Test - working-directory: test/csharp-tableau-loader - run: dotnet test --nologo --logger "console;verbosity=normal" + run: python3 make.py test --lang csharp diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index d7e430d8..4c6df0fe 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -15,11 +15,11 @@ permissions: jobs: test: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - protobuf-version: ["32.0", "3.19.3"] - name: test (${{ matrix.os }}, protobuf ${{ matrix.protobuf-version }}) + name: test (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 10 @@ -29,6 +29,9 @@ jobs: with: submodules: recursive + - name: Read pinned versions + uses: ./.github/actions/load-versions + - name: Install Go uses: actions/setup-go@v6 with: @@ -39,20 +42,25 @@ jobs: - name: Install Buf uses: bufbuild/buf-action@v1 with: - version: 1.67.0 + version: ${{ env.BUF_VERSION }} setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Generate protoconf - working-directory: test/go-tableau-loader - run: buf generate .. - - - name: Vet - run: go vet ./... + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" - - name: Unittest - run: go test -v -timeout 30m -race ./... -coverprofile=coverage.txt - -covermode=atomic + - name: Test + # `make.py test --lang go` runs `buf generate` then `go test`, which + # transitively vets every package (compilation = type-check + vet). + # Plugin-only `go vet` is covered by devcontainer-smoke.yml's + # `make.py test --lang go --smoke` step. + # Explicit --race so Windows CI matches Linux (Windows runners have + # MSVC available; -race needs cgo + a C compiler). On a fresh + # Windows dev machine without MSVC, make.py's default is --no-race; + # CI overrides that. + run: python3 make.py test --lang go --race --coverage - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 diff --git a/.github/workflows/testing-make.yml b/.github/workflows/testing-make.yml new file mode 100644 index 00000000..d89f48d7 --- /dev/null +++ b/.github/workflows/testing-make.yml @@ -0,0 +1,50 @@ +name: Testing make.py + +# Regression tests for make.py itself: unit tests + dry-run snapshot +# tests of the orchestrator. Catches make.py bugs before the slower +# testing-{go,cpp,csharp}.yml E2E workflows even start. + +on: + push: + branches: [master, main] + paths: + - "make.py" + - "test_make.py" + - ".devcontainer/versions.env" + - ".github/workflows/testing-make.yml" + pull_request: + paths: + - "make.py" + - "test_make.py" + - ".devcontainer/versions.env" + - ".github/workflows/testing-make.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 5 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: Run make.py tests + run: python -m pytest test_make.py -v diff --git a/.gitignore b/.gitignore index 09abcd07..58e33296 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ obj/ node_modules *.pb.* +__pycache__/ +.pytest_cache cmd/protoc-gen-cpp-tableau-loader/protoc-gen-cpp-tableau-loader cmd/protoc-gen-go-tableau-loader/protoc-gen-go-tableau-loader @@ -50,3 +52,15 @@ test/csharp-tableau-loader/protoconf # C# Dev Kit language service cache (VS Code) *.lscache +# vcpkg install tree (build artifact, never checked in). +vcpkg_installed/ + +# vcpkg manifest is rendered on the fly by .github/workflows/testing-cpp.yml. +# Local users who want to commit a manifest can `git add -f vcpkg.json`. +test/cpp-tableau-loader/vcpkg.json + +.claude/settings.local.json + +# .NET SDK 10 file-based app / runfile discovery cache (auto-generated by dotnet CLI / C# Dev Kit) +dotnet/runfile-discovery/ + diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 5f237432..00000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "third_party/_submodules/protobuf"] - path = third_party/_submodules/protobuf - url = https://github.com/protocolbuffers/protobuf - ignore = dirty diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..25bd1e30 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,145 @@ +# CLAUDE.md + +Guidance for Claude Code (claude.ai/code) when working in this repository. + +## What this repo is + +`github.com/tableauio/loader` is the official config-loader generator for [Tableau](https://github.com/tableauio/tableau). It ships **three `protoc` plugins** (Go) that read protobuf files annotated with the `tableau.workbook` / `tableau.worksheet` / `tableau.field` extensions and emit strongly-typed loader code: + +| Plugin (under `cmd/`) | Output | Generated extension | +| --- | --- | --- | +| `protoc-gen-go-tableau-loader` | Go | `*.pc.go` | +| `protoc-gen-cpp-tableau-loader` | C++17 | `*.pc.h` / `*.pc.cc` | +| `protoc-gen-csharp-tableau-loader` | C# (Unity 2022.3 LTS / .NET 8) | `*.pc.cs` | + +Generated code is opinionated: every worksheet message becomes a `Messager` with `Load`/`Store`/`Get*`/index/ordered-map accessors; all messagers register into a singleton-ish `Hub`; runtime delegates file IO + protobuf (un)marshaling to `github.com/tableauio/tableau`'s `format`/`load`/`store` packages. + +## Common commands + +Build/test happens **per language** under `test/-tableau-loader/`. The repo root only hosts the Go module + plugin sources; `go test ./...` from root only exercises shared packages (`internal/index`, `internal/loadutil`, `pkg/treemap`, `pkg/udiff`). + +The single cross-platform driver is **`make.py`** (Python 3.10+, stdlib only). It works on Windows, macOS, Linux, and inside the devcontainer, and is what CI calls. + +```sh +python3 make.py setup --lang all # one-time host toolchain install (no-op in container) +python3 make.py generate --lang go # buf generate .. +python3 make.py build --lang cpp +python3 make.py test --lang go +python3 make.py test --lang cpp -k HubTest.Load +python3 make.py env # diagnostic JSON +python3 make.py --version +``` + +C++ wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` before regenerating (gitignored `*.pb.*` shadows fresh codegen). `--no-clean` skips it. A leftover `vcpkg.json` from a previous `--protobuf-version` run is auto-removed in classic mode so cmake doesn't accidentally re-enter manifest mode. + +On Windows, `make.py` wraps every C++ subprocess in `cmd /c "call vcvarsall.bat x64 >nul && "` so MSVC env lives per-subprocess; the shell PATH is never mutated. + +`make.py setup` pins every toolchain dimension (matches CI + devcontainer): Go via official go.dev tarball to `~/.local/go/`, buf via GitHub release binary, protobuf via **vcpkg at the active variant's baseline commit on every host (macOS/Linux/Windows)**, .NET via Microsoft repo (Linux) or Homebrew (macOS) or winget (Windows), cmake/ninja via the host's package manager. Resolved paths cached in `~/.loader-env.json` so subsequent `make.py test --lang cpp` invocations pick them up without re-running setup. + +### Dev container + +- `.devcontainer/` → **Dev Containers: Reopen in Container**. Ubuntu 24.04 + all toolchains pinned. First build ~25 min; reopens instant. +- Inside: `python3 make.py setup` is a no-op. `python3 make.py test --lang ` works for all languages — Dockerfile presets `CMAKE_PREFIX_PATH=/opt/vcpkg/active`. +- Override protobuf version: `LOADER_DEFAULT_VARIANT=legacy-v3 code .` then **Rebuild Container** (switches both protobuf and vcpkg baseline atomically). Variants are declared as `_PROTOBUF_VERSION` + `_VCPKG_BASELINE_COMMIT` pairs in `.devcontainer/versions.env`. For a surgical override of just the protobuf version, `LOADER_PROTOBUF_VERSION=X.Y.Z` still works. +- Single source of truth for all toolchain versions: **`.devcontainer/versions.env`**. + +CI primary tests (`testing-{cpp,go,csharp}.yml`) use `lukka/run-vcpkg` for cached vcpkg installs + `python3 make.py test --lang ` for build/test. `devcontainer-smoke.yml` builds the image on `.devcontainer/**` PRs (amd64 + arm64). `testing-make.yml` runs the make.py unit + dry-run regression suite on every push. + +### Plugin development (Go module at repo root) + +```sh +go vet ./... +go test ./... # internal/index, internal/loadutil, pkg/treemap, pkg/udiff +go test ./internal/index -run Test_ParseIndexDescriptor # single test +go build -o /tmp/p ./cmd/protoc-gen-go-tableau-loader # smoke-build a plugin +``` + +Plugins are invoked through `buf generate` from a test directory (`buf.gen.yaml` runs them via `local: ["go", "run", "../../cmd/protoc-gen-go-tableau-loader"]`), so plugin changes take effect on the next `buf generate` without an explicit install step. + +### Per-language + +```sh +# Go +python3 make.py test --lang go # full +python3 make.py test --lang go -k Test_ActivityConf_OrderedMap # filter +python3 make.py test --lang go --smoke # plugin-only `go vet` (devcontainer-smoke) +python3 make.py test --lang go --coverage # CI: -coverprofile=coverage.txt -covermode=atomic +python3 make.py test --lang go --race # opt in to -race (default off on Windows; needs cgo+MSVC) + +# C++ (requires matching protoc + libprotobuf — protobuf v22+ enforces gencode/runtime check) +python3 make.py test --lang cpp # full +python3 make.py test --lang cpp -k HubTest.Load # filter +python3 make.py test --lang cpp --cxx-std 20 # C++20 +python3 make.py test --lang cpp --cxx-compiler clang # clang++ +python3 make.py test --lang cpp --protobuf-version 3.21.12 # legacy v3 (vcpkg manifest mode) + +# C# +python3 make.py test --lang csharp # full +python3 make.py test --lang csharp -k HubTest.Load # FullyQualifiedName~HubTest.Load +``` + +GoogleTest is fetched via CMake `FetchContent` — no manual install. + +### make.py regression tests + +```sh +pip install pytest && python -m pytest test_make.py -v +``` + +`test_make.py` (next to `make.py`) covers: pure-logic unit tests (`Versions`, `Platform`, `windows_msvc_wrap`, `_winquote`, `Runner`, repo-root discovery) + dry-run snapshot tests (assert exact subprocess sequences for every ` --lang ` combo). Runs in <5s; CI: `.github/workflows/testing-make.yml` (ubuntu/macos/windows). + +## Big-picture architecture + +### Plugin pipeline (the part you'll modify most) + +Every plugin's `main.go` is the same shape: parse flags, set protogen feature bits (proto2 → editions 2024, FEATURE_PROTO3_OPTIONAL), iterate `gen.Files`, decide what to generate per file/message. Decision logic is centralized in **`internal/options`**: + +- `options.NeedGenFile(f)` — file-level gate: must have `(tableau.workbook)` set and at least one message with `(tableau.worksheet)`. +- `options.NeedGenOrderedMap` / `NeedGenIndex` / `NeedGenOrderedIndex` — message-level gates that *also* honour the `lang_options` map on `WorksheetOptions` (e.g. `lang_options: { key: "Index" value: "go" }` = "only generate index accessors for Go"). Language IDs (`cpp`, `go`, `cs`) are in `internal/options/options.go`. + +Each plugin splits work between two passes: + +1. **Per-message generation** (`messager.go` per plugin) — emits one `*.pc.{go,h,cc,cs}` per `.proto`. Delegates ordered-map field/method emission to `cmd//orderedmap/`, index emission to `cmd//indexes/`. **The shared semantic model — what the index syntax means — lives in `internal/index`, not per-plugin**: `ParseIndexDescriptor` walks the message tree and returns a `LevelMessage` linked-list describing what indices apply at each nesting level (map → list → map → list, etc.). Plugins consume this descriptor; do not duplicate parsing per-language. +2. **Cross-message ("embed") generation** — emits `hub.pc.*`, `messager_container.pc.go`, `util.pc.*`. Driven by `cmd//embed.go`, which `//go:embed`s templates under `cmd//embed/templates/` (Go) or `cmd//embed/` (C++/C# also include verbatim `*.pc.{h,cc,cs}` files emitted unchanged). The "all messagers in source order" iteration source is **`internal/xproto.ParseProtoFiles`**. + +C++ plugin sharding: `--shards=N` in `buf.gen.yaml` makes `xproto.ProtoFiles.SplitShards(N)` partition messagers across N `hub_shard*.pc.cc` files to parallelize the (heavy) compile. Tri-state `--mode` (`default` / `hub` / `messager`) lets users split protoconf generation across separate `buf generate` invocations. + +### Index syntax (`internal/index/index.go`) + +`worksheet.index` / `worksheet.ordered_index` strings use a compact mini-language parsed by one regex: + +``` +ID # single-column index on field "ID" +ID@Item # named "Item" +ID@Item # sort within group by SortedCol +(ID, Name)@Item # composite index +CountryItemAttrName # CamelCase concatenation of nested-field path Country.Item.Attr.Name +``` + +Multi-level nested maps/lists are flattened: `CountryItemAttrName` reaches into `country_list[].item_map[].attr_list[].name`. Generated indexes return leveled key structs whose names are derived in `helper.ParseLeveledMapPrefix` — a 3-level map with indexes only at the 2nd level produces fewer key structs than one with indexes at every level (compare `Fruit5Conf` vs `Fruit4Conf` in `test/proto/index_conf.proto`). + +### Hub and Messager runtime (Go) + +- Hub state is held in `atomic.Pointer[MessagerContainer]` so `Load(...)` swaps the snapshot atomically while `Get*()` callers race-freely read the previous one. +- Container is generated (`messager_container.pc.go`) with both generic `GetMessager(name)` and typed `Get()`. `Hub.NewContext` / `FromContext` propagate the snapshot through `context.Context`. +- `hub.WithMutableCheck` is opt-in: enabling it flips `enableBackup()` on each messager, then a goroutine periodically `proto.Equal`s `originalMessage()` vs `Message()` and calls `OnMutate(name, original, current)` (default handler prints a unified diff via `pkg/udiff`). +- Two extension points: `processAfterLoad` (per-messager, runs as `Load` finishes) and `ProcessAfterLoadAll(hub *Hub)` (cross-messager, against a temporary hub). `test/go-tableau-loader/customconf/custom_item_conf.go` shows the canonical pattern: hand-written messager registers via `tableau.Register(func() Messager { ... })` and uses `ProcessAfterLoadAll` to consume data from another messager — that's how derived/computed configs are built. + +### Test data flow + +1. `test/proto/*.proto` — hand-written annotated protos (source of truth for what generators are exercised). +2. `test/testdata/conf/*.json`, `patchconf/`, `patchconf2/`, `patchresult/` — JSON inputs the upstream `tableau` toolchain produces from spreadsheets; loader tests read them at runtime. +3. Each `test/-tableau-loader/` runs `buf generate ..` → language-specific stubs → native test runner against `test/testdata/`. + +Patch tests verify three semantics defined upstream (merge / replace / recursive_patch) — loader's job is just to wire `patch_paths`/`patch_dirs` through `load.MessagerOptions`. + +## Versioning and releases + +Each plugin has its own `version` constant in its `main.go`, released independently via tags shaped `cmd/protoc-gen-{go,cpp,csharp}-tableau-loader/`. Matching workflows in `.github/workflows/release-*.yml` build cross-platform binaries and attach them to the GitHub release. Bump `const version = "..."` in lockstep with the tag. + +## Style and conventions + +- C++ / proto: `clang-format` per `.clang-format` (Google base, 120-col). +- Go: standard `gofmt` / `go vet`; CI runs `go vet ./...` and `go test -race`. +- Generated files end in `.pc.` (the `extensions.PC` constant). Anything matching `*.pb.*` is `.gitignore`d — never commit generated proto-runtime files. +- Worksheet language gating: when adding a feature only some target languages support, gate it via `lang_options` in `internal/options` rather than baking the rule into per-plugin `messager.go`. diff --git a/README.md b/README.md index c6b88d43..ade58207 100644 --- a/README.md +++ b/README.md @@ -2,148 +2,76 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). -## Prerequisites - -> TODO: [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) - -- C++ standard: at least C++17 -- Prepare and init: - - macOS or Linux: `bash init.sh` - - Windows: - 1. Run `prepare.bat` **as Administrator** to automatically install all build dependencies ([Chocolatey](https://chocolatey.org/), [CMake](https://github.com/Kitware/CMake/releases), [Ninja](https://ninja-build.org/), MSVC build tools, and [buf](https://buf.build/)), configure `PATH`, and initialize the MSVC compiler environment: - ```bat - .\prepare.bat - ``` - > ⚠️ **Admin required:** This script uses Chocolatey and MSI installers that write to system-protected directories (`C:\ProgramData`, `C:\Program Files`). Right-click Command Prompt → **Run as administrator**, then execute the script. - > - > Preview what the script would do without making any changes: - > ```bat - > .\prepare.bat --dry-run - > ``` - 2. Run `init.bat` to initialize submodules and build protobuf: - ```bat - .\init.bat - ``` - > **Note:** The **installation** part of `prepare.bat` only runs once per machine — it detects already-installed tools (Chocolatey, Ninja, CMake, MSVC Build Tools, buf) and skips them, so no manual installation is required. - > - > However, the MSVC compiler environment (`cl.exe` on `PATH`, plus `INCLUDE` / `LIB` / `LIBPATH` / `WindowsSdkDir` / `VCToolsInstallDir`) is exported to the **current cmd session only** — `vcvarsall.bat` does not (and should not) write these into the persistent user `PATH`. You therefore need to re-run `.\prepare.bat` in **every new cmd window** before invoking `init.bat` or building the loader. Subsequent runs are near-instant since no installation work is repeated. - -> **Fast path (idempotent re-runs):** Building protobuf takes 5–15 minutes. To make repeated runs cheap, both `init.sh` and `init.bat` short-circuit and exit immediately when `third_party/_submodules/protobuf/.build/_install` already contains a valid `protobuf-config.cmake` (the marker that the previous build finished). This means: -> - Re-running `init.sh` / `init.bat` after a successful first run is a no-op (a second or two). -> - CI workflows cache `.build/_install` (see `.github/workflows/testing-cpp.yml`) and the fast path then turns the "build protobuf" step into a near-instant cache restore. -> - To force a clean rebuild (e.g. after changing protobuf flags or switching `PROTOBUF_REF` to a version whose previously-installed artefacts are still around), set `FORCE_REBUILD_PROTOBUF=1`: -> ```sh -> FORCE_REBUILD_PROTOBUF=1 bash init.sh # macOS / Linux -> ``` -> ```bat -> set FORCE_REBUILD_PROTOBUF=1 && .\init.bat :: Windows (cmd) -> ``` -> Or simply delete `third_party/_submodules/protobuf/.build/` before rerunning. - -### References - -- [Chocolatey](https://chocolatey.org/) -- [CMake 3.31.8](https://github.com/Kitware/CMake/releases/tag/v3.31.8) -- [Ninja](https://ninja-build.org/) -- [Visual Studio 2022](https://visualstudio.microsoft.com/downloads/) -- [Use the Microsoft C++ Build Tools from the command line](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170) -- [buf CLI](https://buf.build/docs/cli/) - -## C++ - -### Dev at Linux - -- Change dir: `cd test/cpp-tableau-loader` -- Generate protoconf: `PATH=../../third_party/_submodules/protobuf/.build/_install/bin:$PATH buf generate ..` -- CMake: - - C++17: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug` - - C++20: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20` - - clang: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=clang++` -- Build: `cmake --build build --parallel` -- Test: `ctest --test-dir build --output-on-failure` - -### Dev at Windows +| Plugin | Language | Generated extension | +| --- | --- | --- | +| `protoc-gen-go-tableau-loader` | Go | `*.pc.go` | +| `protoc-gen-cpp-tableau-loader` | C++17 | `*.pc.h` / `*.pc.cc` | +| `protoc-gen-csharp-tableau-loader` | C# (Unity 2022.3 LTS / .NET 8) | `*.pc.cs` | -> **Important:** CMake with Ninja requires MSVC environment variables (`cl.exe`, `INCLUDE`, `LIB`, etc.) to be active. Run `.\prepare.bat` from the **loader** root in the **same cmd session** (use **cmd**, not PowerShell — `prepare.bat` exports vars via `endlocal & set ...` which only works for a cmd parent process) before switching to the test directory. Opening a new terminal window will lose these variables. -> -> **Build type:** The protobuf submodule is built as **Debug** (`/MTd`) by `init.bat`. To avoid LNK2038 `_ITERATOR_DEBUG_LEVEL` / `RuntimeLibrary` CRT-mismatch errors, the loader must also be built as Debug. `CMakeLists.txt` does not set a default, so always pass `-DCMAKE_BUILD_TYPE=Debug` explicitly — also required for multi-config generators (Visual Studio default = Debug, but stay explicit to match the cached protobuf). +## Quick start -- Initialize MSVC environment (from loader root): `.\prepare.bat` -- Change dir: `cd test\cpp-tableau-loader`, or change directory with Drive, e.g.: `cd /D D:\GitHub\loader\test\cpp-tableau-loader` -- Generate protoconf: - - cmd: `cmd /C "set PATH=..\..\third_party\_submodules\protobuf\.build\_install\bin;%PATH% && buf generate .."` - - PowerShell: `$env:PATH = "..\..\third_party\_submodules\protobuf\.build\_install\bin;" + $env:PATH; buf generate ..` -- CMake: - - C++17: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug` - - C++20: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20` -- Build: `cmake --build build --parallel` -- Test: `ctest --test-dir build --output-on-failure` +Use [`make.py`](./make.py) (Python 3.10+, stdlib only): -> **Note:** Tests are written with [GoogleTest](https://github.com/google/googletest), pulled in via CMake `FetchContent` (no manual installation needed). +```sh +python3 make.py setup --lang all # one-time host toolchain install +python3 make.py test --lang go # Go +python3 make.py test --lang cpp # C++ +python3 make.py test --lang csharp # C# +python3 make.py test --lang ts # TypeScript (experimental) +``` -### References +Recommended environment: [devcontainer](./.devcontainer/) (open in VS Code → **Dev Containers: Reopen in Container**). Inside the container, `setup` is a no-op. -- [Protocol Buffers C++ Installation](https://github.com/protocolbuffers/protobuf/tree/master/src) -- [Protocol Buffers C++ Reference](https://protobuf.dev/reference/cpp/) - -## Go - -- Install: **go1.24** or above -- Change dir: `cd test/go-tableau-loader` -- Generate protoconf: `buf generate ..` -- Test: `go test ./...` +Native hosts: `python3 make.py setup` installs everything pinned to [`./.devcontainer/versions.env`](./.devcontainer/versions.env) — the same versions CI and the devcontainer use. Toolchain layout per host: -### References - -- [Protocol Buffers Go Reference](https://protobuf.dev/reference/go/) +- **Go** — official tarball from go.dev to `~/.local/go/` (Linux/macOS) or winget (Windows). +- **buf** — pinned binary from GitHub releases to `~/.local/bin/` (Linux/macOS) or `%LOCALAPPDATA%\buf\bin\` (Windows). +- **protobuf** — vcpkg at `VCPKG_BASELINE_COMMIT` on every native host (Linux/macOS/Windows). Switch versions per-test with `--protobuf-version`. +- **.NET / Node / cmake / ninja** — Homebrew (macOS), Microsoft+NodeSource apt repos (Linux), winget+Chocolatey (Windows). -## C# +On Windows, run setup from **cmd as Administrator** the first time. Subsequent commands work from any shell because each subprocess sources `vcvarsall.bat` itself — your shell PATH/INCLUDE/LIB are never mutated. -### Requirements +## Commands -- Unity 2022.3 LTS (C# 9) -- dotnet-sdk-8.0 +```sh +python3 make.py setup [--lang go|cpp|csharp|ts|all] +python3 make.py generate --lang go|cpp|csharp|ts +python3 make.py build --lang go|cpp|csharp|ts [--cxx-std 17|20] [--cxx-compiler msvc|clang|gcc] + [--protobuf-version ] [--triplet ] +python3 make.py test --lang go|cpp|csharp|ts [-k ] [--smoke] [--coverage] [--no-race] + (+ all build flags) +python3 make.py clean [--lang ...] [--all] +python3 make.py env # diagnostic JSON +python3 make.py --version +``` -### Test +Global flags: `--verbose / -v`, `--dry-run`, `--cwd `. -- Install: **dotnet-sdk-8.0** -- Change dir: `cd test/csharp-tableau-loader` -- Generate protoconf: - - macOS / Linux: `PATH=../../third_party/_submodules/protobuf/.build/_install/bin:$PATH buf generate ..` - - Windows (cmd): `cmd /C "set PATH=..\..\third_party\_submodules\protobuf\.build\_install\bin;%PATH% && buf generate .."` - - Windows (PowerShell): `$env:PATH = "..\..\third_party\_submodules\protobuf\.build\_install\bin;" + $env:PATH; buf generate ..` -- Test: `dotnet test` +Examples: -> **Note:** Tests are written with [xUnit](https://xunit.net/). +```sh +python3 make.py test --lang go -k Test_ActivityConf_OrderedMap +python3 make.py test --lang go --race # opt in (Windows default is off; needs cgo+MSVC) +python3 make.py test --lang cpp --protobuf-version 3.21.12 +python3 make.py test --lang csharp -k HubTest.Load +python3 make.py test --lang cpp --no-clean # skip pre-build wipe +``` -## TypeScript +The C++ flow wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` by default (gitignored `*.pb.*` from a prior protobuf version shadows fresh codegen). -### Requirements +## Tests for `make.py` itself -- nodejs v16.0.0 -- protobufjs v7.2.3 +```sh +pip install pytest +python -m pytest test_make.py -v +``` -### Test +CI: [`.github/workflows/testing-make.yml`](.github/workflows/testing-make.yml). -- Change dir: `cd test/ts-tableau-loader` -- Install depedencies: `npm install` -- Generate protoconf: `npm run generate` -- Test: `npm run test` +## References -### Problems in [protobufjs](https://github.com/protobufjs/protobuf.js): - -- [Unable to use Google well known types](https://github.com/protobufjs/protobuf.js/issues/1042) -- [google.protobuf.Timestamp deserialization incompatible with canonical JSON representation](https://github.com/protobufjs/protobuf.js/issues/893) -- [Implement wrapper for google.protobuf.Timestamp, and correctly generate wrappers for static target.](https://github.com/protobufjs/protobuf.js/pull/1258) - - -> [protobufjs: Reflection vs. static code](https://github.com/protobufjs/protobuf.js/blob/master/cli/README.md#reflection-vs-static-code) - -If using reflection (`.proto` or `JSON`) but not static code, and for well-known types support, then [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) is a good option. This library implements proto3 JSON serialization and deserialization for -[protobuf.js](https://www.npmjs.com/package/protobufjs) protobuf objects -according to the [spec](https://protobuf.dev/programming-guides/proto3/#json). - -### References: - -- [How to Setup a TypeScript + Node.js Project](https://khalilstemmler.com/blogs/typescript/node-starter-project/) -- [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) +- [Protocol Buffers C++ Reference](https://protobuf.dev/reference/cpp/) +- [Protocol Buffers Go Reference](https://protobuf.dev/reference/go/) +- [protobuf-es](https://github.com/bufbuild/protobuf-es) +- [vcpkg](https://github.com/microsoft/vcpkg) +- [buf CLI](https://buf.build/docs/cli/) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index dc8d06de..b586f9ac 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -63,7 +63,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::pa const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + const tableau::WorksheetOptions& worksheet_options = util::GetExtension(descriptor->options(), tableau::worksheet); if (worksheet_options.patch() != tableau::PATCH_NONE) { return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index 40821ae3..c570d17d 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -63,6 +63,21 @@ const std::string& Format2Ext(Format fmt) { #undef GetMessage #endif +// MapKeyFd returns the key FieldDescriptor of a map-entry message. +// +// ``Descriptor::map_key()`` is a convenience wrapper added in protobuf +// v3.12.0; on older runtimes we fall back to ``field(0)``, which is +// equivalent because map-entry messages always synthesize the key as +// field 0 (and value as field 1) per the proto3 wire-format contract. +inline const google::protobuf::FieldDescriptor* MapKeyFd( + const google::protobuf::Descriptor* map_entry) { +#if GOOGLE_PROTOBUF_VERSION < 3012000 + return map_entry->field(0); +#else + return map_entry->map_key(); +#endif +} + // PatchMessage patches src into dst, which must be a message with the same descriptor. // // # Default PatchMessage mechanism @@ -105,7 +120,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag // Iterates over every populated field. for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + const tableau::FieldOptions& opts = GetExtension(fd->options(), tableau::field); tableau::Patch patch = opts.prop().patch(); if (patch == tableau::PATCH_REPLACE) { dst_reflection->ClearField(&dst, fd); @@ -113,7 +128,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag if (fd->is_map()) { // Reference: // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); + auto key_fd = MapKeyFd(fd->message_type()); int src_count = src_reflection->FieldSize(src, fd); int dst_count = dst_reflection->FieldSize(dst, fd); switch (key_fd->cpp_type()) { diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 574848c2..8616839e 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -5,6 +5,7 @@ #include #include #include +#include // Protobuf versions before v4.22.0 (GOOGLE_PROTOBUF_VERSION < 4022000) use the legacy // logging interface (LogLevel, SetLogHandler). Newer versions removed it in favor of Abseil logging. @@ -68,6 +69,29 @@ const std::string& Format2Ext(Format fmt); // PatchMessage patches src into dst, which must be a message with the same descriptor. bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); +// GetExtension reads a typed custom-option extension off any *Options +// message (e.g. MessageOptions, FieldOptions). +// +// On protobuf runtimes < v3.15.0 a static-initialization-order quirk can +// leave the extension payload parked in the options' unknown_fields, so +// `options.GetExtension(id)` returns the default instance. We work around +// it by serializing and reparsing into a fresh OptionsT at call time, by +// which point all extension registrations have run. +// Reference: https://github.com/protocolbuffers/protobuf/releases/tag/v3.15.0 +template +inline auto GetExtension(const OptionsT& options, const ExtT& id) + -> typename std::decay::type { +#if GOOGLE_PROTOBUF_VERSION < 3015000 + OptionsT reparsed; + std::string buf; + options.SerializeToString(&buf); + reparsed.ParseFromString(buf); + return reparsed.GetExtension(id); +#else + return options.GetExtension(id); +#endif +} + #if TABLEAU_PB_LOG_LEGACY // ProtobufLogHandler redirects protobuf internal logs to tableau logger. // Only available for protobuf < v4.22.0, as newer versions removed the old logging interface. diff --git a/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go b/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go index 772b6f86..4119dbe3 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go +++ b/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go @@ -29,19 +29,23 @@ func escapeIdentifier(str string) string { return str } +// cppKeywords mirrors protoc's C++ codegen reserved-identifier list +// (kKeywordList). Keeping them in sync ensures we escape identifiers exactly +// the way protoc does (append a trailing underscore on collision), so the +// names we emit match the generated *.pb.h symbols. +// // Ref: // -// https://en.cppreference.com/w/cpp/keyword +// https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/compiler/cpp/helpers.cc func init() { cppKeywords = map[string]bool{ + "NULL": true, "alignas": true, "alignof": true, "and": true, "and_eq": true, "asm": true, - "atomic_cancel": true, - "atomic_commit": true, - "atomic_noexcept": true, + "assert": true, "auto": true, "bitand": true, "bitor": true, @@ -50,21 +54,12 @@ func init() { "case": true, "catch": true, "char": true, - "char8_t": true, - "char16_t": true, - "char32_t": true, "class": true, "compl": true, - "concept": true, "const": true, - "consteval": true, "constexpr": true, - "constinit": true, "const_cast": true, "continue": true, - "co_await": true, - "co_return": true, - "co_yield": true, "decltype": true, "default": true, "delete": true, @@ -99,9 +94,7 @@ func init() { "protected": true, "public": true, "register": true, - "reflexpr": true, "reinterpret_cast": true, - "requires": true, "return": true, "short": true, "signed": true, @@ -111,7 +104,6 @@ func init() { "static_cast": true, "struct": true, "switch": true, - "synchronized": true, "template": true, "this": true, "thread_local": true, @@ -131,5 +123,15 @@ func init() { "while": true, "xor": true, "xor_eq": true, + "char8_t": true, + "char16_t": true, + "char32_t": true, + "concept": true, + "consteval": true, + "constinit": true, + "co_await": true, + "co_return": true, + "co_yield": true, + "requires": true, } } diff --git a/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go b/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go index d69e7380..807579db 100644 --- a/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go +++ b/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go @@ -43,7 +43,19 @@ func escapeIdentifier(str string) string { // csharpKeywords is the set of C# reserved keywords used by escapeIdentifier // to detect and escape naming conflicts. // -// Ref: https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/keywords +// Note on protobuf parity: protobuf's own C# generator does NOT keep a C# +// language-keyword list, because it emits PascalCase property names that never +// collide with the (lower-case) C# keywords; it only guards against its own +// generated member names (e.g. Types/Descriptor/Equals/Parser...) and the +// containing type name, appending "_" on collision (see GetPropertyName in +// csharp_helpers.cc). This loader instead emits lowerCamelCase parameter +// identifiers, which CAN collide with C# keywords, so it maintains the language +// keyword set itself. The authoritative source for that set is the C# spec. +// +// Ref: +// +// https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/compiler/csharp/csharp_helpers.cc (GetPropertyName) +// https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/keywords func init() { csharpKeywords = map[string]bool{ "abstract": true, diff --git a/cmd/protoc-gen-go-tableau-loader/helper/keyword.go b/cmd/protoc-gen-go-tableau-loader/helper/keyword.go index e71d22f7..e0d7377d 100644 --- a/cmd/protoc-gen-go-tableau-loader/helper/keyword.go +++ b/cmd/protoc-gen-go-tableau-loader/helper/keyword.go @@ -1,14 +1,22 @@ package helper import ( + "go/token" "strings" "unicode" "github.com/iancoleman/strcase" ) -var golangKeywords map[string]bool - +// escapeIdentifier converts a raw string into a valid Go lowerCamelCase +// identifier, escaping Go reserved words the same way protoc-gen-go's +// GoSanitized does: it consults go/token's keyword table via +// token.Lookup(...).IsKeyword(). On collision a trailing "_" is appended. +// +// Ref: +// +// https://github.com/protocolbuffers/protobuf-go/blob/master/internal/strs/strings.go (GoSanitized) +// https://pkg.go.dev/go/token#Lookup func escapeIdentifier(str string) string { // Filter invalid runes var result strings.Builder @@ -24,49 +32,13 @@ func escapeIdentifier(str string) string { if len(str) != 0 && unicode.IsDigit(rune(str[0])) { str = "_" + str } - // Avoid go keywords - if _, ok := golangKeywords[str]; ok { + // Avoid Go keywords, plus the loader-specific reserved name "x": the + // generated Go code uses "x" as the method receiver name (e.g., + // `func (x *FooConf) FindIndex1(...)`). If a proto field named "X" is used as + // an index key, escapeIdentifier converts it to "x" (lowerCamelCase), which + // would shadow the receiver and cause a compile error, so it is escaped too. + if token.Lookup(str).IsKeyword() || str == "x" { return str + "_" } return str } - -// Ref: -// -// https://go.dev/ref/spec#Keywords -func init() { - golangKeywords = map[string]bool{ - "break": true, - "case": true, - "chan": true, - "const": true, - "continue": true, - "default": true, - "defer": true, - "pi": true, - "else": true, - "fallthrough": true, - "for": true, - "func": true, - "go": true, - "goto": true, - "if": true, - "import": true, - "interface": true, - "map": true, - "package": true, - "range": true, - "return": true, - "select": true, - "struct": true, - "switch": true, - "type": true, - "var": true, - // "x" is treated as a keyword because the generated Go code uses "x" as the - // method receiver name (e.g., `func (x *FooConf) FindIndex1(...)`). If a proto - // field named "X" is used as an index key, escapeIdentifier converts it to "x" - // (lowerCamelCase), which would shadow the receiver and cause a compile error. - // By treating "x" as a keyword, it gets escaped to "x_" to avoid the conflict. - "x": true, - } -} diff --git a/go.mod b/go.mod index 10bf1e80..d2625c9a 100644 --- a/go.mod +++ b/go.mod @@ -5,34 +5,41 @@ go 1.24.0 require ( github.com/aymanbagabas/go-udiff v0.2.0 github.com/iancoleman/strcase v0.3.0 - github.com/stretchr/testify v1.10.0 - github.com/tableauio/tableau v0.15.1 + github.com/stretchr/testify v1.11.1 + github.com/tableauio/tableau v0.16.0 google.golang.org/protobuf v1.36.11 ) require ( - github.com/antchfx/xpath v0.0.0-20170515025933-1f3266e77307 // indirect - github.com/bufbuild/protocompile v0.10.0 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect + buf.build/go/protovalidate v1.2.0 // indirect + cel.dev/expr v0.25.1 // indirect + github.com/antchfx/xpath v1.3.6 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/bufbuild/protocompile v0.14.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/google/cel-go v0.28.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20240820135758-21b1d9897dc7 // indirect - github.com/richardlehane/mscfb v1.0.4 // indirect - github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/richardlehane/mscfb v1.0.6 // indirect + github.com/richardlehane/msoleps v1.0.6 // indirect github.com/subchen/go-xmldom v1.1.2 // indirect - github.com/valyala/fastjson v1.6.7 // indirect - github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect - github.com/xuri/excelize/v2 v2.9.0 // indirect - github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/text v0.19.0 // indirect + github.com/tiendc/go-deepcopy v1.7.2 // indirect + github.com/valyala/fastjson v1.6.10 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/excelize/v2 v2.10.1 // indirect + github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.1 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 // indirect + golang.org/x/net v0.50.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 08feefe5..8a8f3c15 100644 --- a/go.sum +++ b/go.sum @@ -1,75 +1,93 @@ -github.com/antchfx/xpath v0.0.0-20170515025933-1f3266e77307 h1:C735MoY/X+UOx6SECmHk5pVOj51h839Ph13pEoY8UmU= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/antchfx/xpath v0.0.0-20170515025933-1f3266e77307/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= +github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/bufbuild/protocompile v0.10.0 h1:+jW/wnLMLxaCEG8AX9lD0bQ5v9h1RUiMKOBOT5ll9dM= -github.com/bufbuild/protocompile v0.10.0/go.mod h1:G9qQIQo0xZ6Uyj6CMNz0saGmx2so+KONo8/KrELABiY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/google/cel-go v0.28.0 h1:KjSWstCpz/MN5t4a8gnGJNIYUsJRpdi/r97xWDphIQc= +github.com/google/cel-go v0.28.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/protocolbuffers/txtpbfmt v0.0.0-20240820135758-21b1d9897dc7 h1:GkKZUEPNgwIk3LK4Er5vxnaNKk1pdjI3Oc6oTBwBsxQ= github.com/protocolbuffers/txtpbfmt v0.0.0-20240820135758-21b1d9897dc7/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c= -github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= -github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= -github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= -github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8= +github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo= +github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg= +github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subchen/go-xmldom v1.1.2 h1:7evI2YqfYYOnuj+PBwyaOZZYjl3iWq35P6KfBUw9jeU= github.com/subchen/go-xmldom v1.1.2/go.mod h1:6Pg/HuX5/T4Jlj0IPJF1sRxKVoI/rrKP6LIMge9d5/8= -github.com/tableauio/tableau v0.15.1 h1:NbTxG7xft57s5SJUXVykNiKWE7/JPKIZsQR0S57hsuE= -github.com/tableauio/tableau v0.15.1/go.mod h1:TjQ0ZZMLQ4lBsFO+pMYdfxhlu68taqtpA4/BlAiqBbI= -github.com/valyala/fastjson v1.6.7 h1:ZE4tRy0CIkh+qDc5McjatheGX2czdn8slQjomexVpBM= -github.com/valyala/fastjson v1.6.7/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= -github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d h1:llb0neMWDQe87IzJLS4Ci7psK/lVsjIS2otl+1WyRyY= -github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= -github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE= -github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A= -github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +github.com/tableauio/tableau v0.16.0 h1:cUSPkTJIHTivTXnEsWH2iGawLeI4cNJnp8BGDkvKKuk= +github.com/tableauio/tableau v0.16.0/go.mod h1:TAlrDUuokQaodTyHAymVRRnqKV00qd5eCuPvl4W0GPg= +github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44= +github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ= +github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= +github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.10.1 h1:V62UlqopMqha3kOpnlHy2CcRVw1V8E63jFoWUmMzxN0= +github.com/xuri/excelize/v2 v2.10.1/go.mod h1:iG5tARpgaEeIhTqt3/fgXCGoBRt4hNXgCp3tfXKoOIc= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE= +golang.org/x/exp v0.0.0-20250813145105-42675adae3e6/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= +golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ= +golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a h1:DMCgtIAIQGZqJXMVzJF4MV8BlWoJh2ZuFiRdAleyr58= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a/go.mod h1:y2yVLIE/CSMCPXaHnSKXxu1spLPnglFLegmgdY23uuE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a h1:tPE/Kp+x9dMSwUm/uM0JKK0IfdiJkwAbSMSeZBXXJXc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/init.bat b/init.bat deleted file mode 100644 index 6d43624f..00000000 --- a/init.bat +++ /dev/null @@ -1,151 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -REM Initialize build environment (installs choco/ninja/MSVC if needed, sets up PATH) -call "%~dp0prepare.bat" -if errorlevel 1 ( - echo [ERROR] prepare.bat failed. Aborting. - exit /b 1 -) - -for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i -cd /d "%repoRoot%" - -REM Initialize only the protobuf submodule (non-recursive). Protobuf's own -REM nested submodules (third_party/googletest, third_party/benchmark on v3.x) -REM are only needed when protobuf_BUILD_TESTS=ON / benchmarks are enabled, and -REM modern protobuf (v4+/v21+) has dropped git submodules entirely in favor of -REM CMake FetchContent. Skipping --recursive saves clone time and CI bandwidth. -git submodule update --init third_party/_submodules/protobuf - -REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -cd third_party\_submodules\protobuf - -REM If PROTOBUF_REF is set, switch submodule to the specified ref -if not "%PROTOBUF_REF%"=="" ( - echo Switching protobuf submodule to %PROTOBUF_REF%... - git fetch --tags - git checkout %PROTOBUF_REF% -) - -REM --------------------------------------------------------------------------- -REM Detect protobuf major version and build the cmake command line. We compute -REM both up-front (before the fast-path check) so the signature comparison -REM below can include the exact cmake invocation we would run. -REM - protobuf v3.x : CMakeLists.txt is in cmake/ subdirectory -REM - protobuf v4+ : CMakeLists.txt is in root directory -REM --------------------------------------------------------------------------- -for /f "tokens=*" %%v in ('git describe --tags --abbrev^=0 2^>nul') do set PROTOBUF_VERSION=%%v -if not defined PROTOBUF_VERSION set PROTOBUF_VERSION=unknown -echo Detected protobuf version: %PROTOBUF_VERSION% - -REM Extract major version number from tag (e.g., v3.19.3 -> 3, v32.0 -> 32) -set "VER_STR=%PROTOBUF_VERSION:~1%" -for /f "tokens=1 delims=." %%a in ("%VER_STR%") do set MAJOR_VERSION=%%a - -if %MAJOR_VERSION% LEQ 3 ( - REM Legacy protobuf (v3.x): CMakeLists.txt is in cmake/ subdirectory - set "PROTOBUF_BUILD_VARIANT=legacy" - set "CMAKE_SRC=cmake" - set "CMAKE_FLAGS=-DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=17 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF" -) else ( - REM Modern protobuf (v4+/v21+/v32+): CMakeLists.txt is in root directory - REM Refer: https://github.com/protocolbuffers/protobuf/blob/v32.0/cmake/README.md#cmake-configuration - REM - protobuf_MSVC_STATIC_RUNTIME defaults to ON, which uses static CRT (/MTd for Debug). - REM Our project's CMakeLists.txt also sets static CRT to match. - REM - protobuf_WITH_ZLIB=OFF: disable ZLIB dependency to avoid ZLIB::ZLIB link requirement - REM in protobuf's exported CMake targets, which simplifies cross-platform builds. - REM - protobuf_BUILD_SHARED_LIBS=OFF: build static libraries explicitly. - set "PROTOBUF_BUILD_VARIANT=modern" - set "CMAKE_SRC=." - set "CMAKE_FLAGS=-DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=17 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dutf8_range_ENABLE_INSTALL=ON" -) - -REM Build a stable, multi-line signature describing the inputs that determine -REM the contents of .build\_install. Any change to these values must -REM invalidate the fast-path. Adding new compile-time inputs? Append a line. -set "SIG_FILE=.build\_install\.build_signature" -set "SIG_LINE_1=schema=1" -set "SIG_LINE_2=version=!PROTOBUF_VERSION!" -set "SIG_LINE_3=variant=!PROTOBUF_BUILD_VARIANT!" -set "SIG_LINE_4=cmake_args=-S !CMAKE_SRC! -B .build -G Ninja !CMAKE_FLAGS!" - -REM --------------------------------------------------------------------------- -REM Fast path: if a previous build's signature file matches the one we're -REM about to use, skip the (very long) protobuf compile entirely. -REM Set FORCE_REBUILD_PROTOBUF=1 to bypass this short-circuit unconditionally. -REM --------------------------------------------------------------------------- -if not "%FORCE_REBUILD_PROTOBUF%"=="" goto :no_fast_path -if not exist "!SIG_FILE!" goto :no_fast_path - -REM Read the file 4 lines at a time and compare with the expected signature. -set "ACTUAL_LINE_1=" -set "ACTUAL_LINE_2=" -set "ACTUAL_LINE_3=" -set "ACTUAL_LINE_4=" -set "_LINE_NO=0" -for /f "usebackq delims=" %%L in ("!SIG_FILE!") do ( - set /a _LINE_NO+=1 - set "ACTUAL_LINE_!_LINE_NO!=%%L" -) - -if not "!ACTUAL_LINE_1!"=="!SIG_LINE_1!" goto :sig_mismatch -if not "!ACTUAL_LINE_2!"=="!SIG_LINE_2!" goto :sig_mismatch -if not "!ACTUAL_LINE_3!"=="!SIG_LINE_3!" goto :sig_mismatch -if not "!ACTUAL_LINE_4!"=="!SIG_LINE_4!" goto :sig_mismatch - -echo [INFO] Build signature matches; reusing existing protobuf install at .build\_install. -echo [INFO] Set FORCE_REBUILD_PROTOBUF=1 to force a clean rebuild. -goto :eof - -:sig_mismatch -echo [INFO] Build signature mismatch; rebuilding protobuf. -echo [INFO] actual: -echo [INFO] !ACTUAL_LINE_1! -echo [INFO] !ACTUAL_LINE_2! -echo [INFO] !ACTUAL_LINE_3! -echo [INFO] !ACTUAL_LINE_4! -echo [INFO] expected: -echo [INFO] !SIG_LINE_1! -echo [INFO] !SIG_LINE_2! -echo [INFO] !SIG_LINE_3! -echo [INFO] !SIG_LINE_4! - -:no_fast_path -REM Wipe any stale install dir so we don't leave half-overwritten files behind -REM when cmake flags change (e.g. Debug -> Release puts artifacts in different -REM places, an in-place re-install would mix old and new). -if exist .build rmdir /s /q .build - -REM --------------------------------------------------------------------------- -REM Configure -REM --------------------------------------------------------------------------- -if "!PROTOBUF_BUILD_VARIANT!"=="legacy" ( - echo Using legacy cmake\ subdirectory for protobuf %PROTOBUF_VERSION% -) else ( - echo Using root CMakeLists.txt for protobuf %PROTOBUF_VERSION% -) -cmake -S !CMAKE_SRC! -B .build -G Ninja !CMAKE_FLAGS! -if errorlevel 1 exit /b 1 - -REM Compile the code -cmake --build .build --parallel -if errorlevel 1 exit /b 1 - -REM Install into .build/_install so that protobuf-config.cmake (along with -REM absl and utf8_range configs) is generated for find_package(Protobuf CONFIG) -REM used by downstream CMakeLists.txt. -REM NOTE: .build/ is already in protobuf's .gitignore, so _install stays clean. -cmake --install .build --prefix .build\_install -if errorlevel 1 exit /b 1 - -REM Persist the signature so the next run can fast-path skip when nothing changed. -> "!SIG_FILE!" ( - echo !SIG_LINE_1! - echo !SIG_LINE_2! - echo !SIG_LINE_3! - echo !SIG_LINE_4! -) -echo [INFO] Wrote build signature to !SIG_FILE! - -endlocal diff --git a/init.sh b/init.sh deleted file mode 100755 index d5b25105..00000000 --- a/init.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# set -eux -set -e -set -o pipefail - -cd "$(git rev-parse --show-toplevel)" - -# Initialize only the protobuf submodule (non-recursive). Protobuf's own nested -# submodules (third_party/googletest, third_party/benchmark on v3.x) are only -# needed when protobuf_BUILD_TESTS=ON / benchmarks are enabled, and modern -# protobuf (v4+/v21+) has dropped git submodules entirely in favor of CMake -# FetchContent. Skipping --recursive saves clone time and CI bandwidth. -git submodule update --init third_party/_submodules/protobuf - -# prerequisites -# On Ubuntu/Debian, you can install them with: -# sudo apt-get install autoconf automake libtool curl make g++ unzip - -# Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -cd third_party/_submodules/protobuf - -# If PROTOBUF_REF is set, switch submodule to the specified ref -if [ -n "${PROTOBUF_REF:-}" ]; then - echo "Switching protobuf submodule to ${PROTOBUF_REF}..." - git fetch --tags - git checkout "${PROTOBUF_REF}" -fi - -# ----------------------------------------------------------------------------- -# Detect protobuf major version and build the cmake command line. We compute -# both up-front (before the fast-path check) so the signature comparison below -# can include the exact cmake invocation we would run. -# - protobuf v3.x : CMakeLists.txt is in cmake/ subdirectory -# - protobuf v4+ : CMakeLists.txt is in root directory -# ----------------------------------------------------------------------------- -PROTOBUF_VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "unknown") -echo "Detected protobuf version: ${PROTOBUF_VERSION}" - -# Extract major version number from tag (e.g., v3.19.3 -> 3, v32.0 -> 32) -MAJOR_VERSION=$(echo "${PROTOBUF_VERSION}" | sed 's/^v//' | cut -d. -f1) - -if [ "${MAJOR_VERSION}" -le 3 ] 2>/dev/null; then - # Legacy protobuf (v3.x): CMakeLists.txt is in cmake/ subdirectory - PROTOBUF_BUILD_VARIANT="legacy" - CMAKE_ARGS=( - -S cmake - -B .build - -G Ninja - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_STANDARD=17 - -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_WITH_ZLIB=OFF - -Dprotobuf_BUILD_SHARED_LIBS=OFF - ) -else - # Modern protobuf (v4+/v21+/v32+): CMakeLists.txt is in root directory - # Refer: https://github.com/protocolbuffers/protobuf/blob/v32.0/cmake/README.md#cmake-configuration - # - protobuf_WITH_ZLIB=OFF: disable ZLIB dependency to avoid ZLIB::ZLIB link requirement - # in protobuf's exported CMake targets, which simplifies cross-platform builds. - # - protobuf_BUILD_SHARED_LIBS=OFF: build static libraries explicitly. - PROTOBUF_BUILD_VARIANT="modern" - CMAKE_ARGS=( - -S . - -B .build - -G Ninja - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_STANDARD=17 - -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_WITH_ZLIB=OFF - -Dprotobuf_BUILD_SHARED_LIBS=OFF - -Dutf8_range_ENABLE_INSTALL=ON - ) -fi - -# Build a stable, multi-line signature describing the inputs that determine -# the contents of .build/_install. Any change to these values must invalidate -# the fast-path. Adding new compile-time inputs? Append a line here. -SIG_FILE=".build/_install/.build_signature" -EXPECTED_SIGNATURE=$(printf '%s\n' \ - "schema=1" \ - "version=${PROTOBUF_VERSION}" \ - "variant=${PROTOBUF_BUILD_VARIANT}" \ - "cmake_args=${CMAKE_ARGS[*]}") - -# ----------------------------------------------------------------------------- -# Fast path: if a previous build's _install dir is present AND its embedded -# signature matches the one we're about to use, skip the (very long) protobuf -# compile entirely. -# Set FORCE_REBUILD_PROTOBUF=1 to bypass this short-circuit unconditionally. -# ----------------------------------------------------------------------------- -if [ -z "${FORCE_REBUILD_PROTOBUF:-}" ] && [ -f "${SIG_FILE}" ]; then - ACTUAL_SIGNATURE=$(cat "${SIG_FILE}") - if [ "${ACTUAL_SIGNATURE}" = "${EXPECTED_SIGNATURE}" ]; then - echo "[INFO] Build signature matches; reusing existing protobuf install at .build/_install." - echo "[INFO] Set FORCE_REBUILD_PROTOBUF=1 to force a clean rebuild." - exit 0 - fi - echo "[INFO] Build signature mismatch; rebuilding protobuf." - echo "[INFO] actual:" - printf '%s\n' "${ACTUAL_SIGNATURE}" | sed 's/^/[INFO] /' - echo "[INFO] expected:" - printf '%s\n' "${EXPECTED_SIGNATURE}" | sed 's/^/[INFO] /' -fi - -# Wipe any stale install dir so we don't leave half-overwritten files behind -# when cmake flags change (e.g. Debug -> Release puts artifacts in different -# places, an in-place re-install would mix old and new). -rm -rf .build 2>/dev/null || true - -# ----------------------------------------------------------------------------- -# Configure -# ----------------------------------------------------------------------------- -if [ "${PROTOBUF_BUILD_VARIANT}" = "legacy" ]; then - echo "Using legacy cmake/ subdirectory for protobuf ${PROTOBUF_VERSION}" -else - echo "Using root CMakeLists.txt for protobuf ${PROTOBUF_VERSION}" -fi -cmake "${CMAKE_ARGS[@]}" - -# Compile the code -cmake --build .build --parallel - -# Install into .build/_install so that protobuf-config.cmake (along with -# absl and utf8_range configs) is generated for find_package(Protobuf CONFIG) -# used by downstream CMakeLists.txt. -# NOTE: .build/ is already in protobuf's .gitignore, so _install stays clean. -cmake --install .build --prefix .build/_install - -# Persist the signature so the next run can fast-path skip when nothing changed. -printf '%s\n' "${EXPECTED_SIGNATURE}" >"${SIG_FILE}" -echo "[INFO] Wrote build signature to ${SIG_FILE}" diff --git a/make.py b/make.py new file mode 100644 index 00000000..37d0cafb --- /dev/null +++ b/make.py @@ -0,0 +1,2005 @@ +#!/usr/bin/env python3 +""" +make.py — single cross-platform entrypoint for the tableauio/loader repo. + +Consolidates per-language `buf generate` / `cmake` / `go test` / `dotnet test` +recipes into one Python tool that works identically on +native Windows, macOS, Linux, and inside the devcontainer. + +Usage (high level): + python3 make.py setup [--lang go|cpp|csharp|all] [--dry-run] + python3 make.py generate --lang go|cpp|csharp + python3 make.py build --lang go|cpp|csharp [build flags] + python3 make.py test --lang go|cpp|csharp [build flags] [-k FILTER] [--smoke] + python3 make.py clean [--lang ...] [--all] + python3 make.py env + python3 make.py --version + +Standard flags (apply to every subcommand): + --verbose / -v echo every subprocess + --dry-run print but do not execute + --cwd repo root (default: auto-detect from versions.env) + +The Windows MSVC env trick: every command that needs cl.exe / vcpkg-protoc +runs through Platform.windows_msvc_wrap(), which transparently wraps the +command in `cmd /c " x64 && "`. The user's shell +PATH/INCLUDE/LIB is never mutated. + +Stdlib only — no `pip install` required. Targets Python >= 3.10. +""" + +import argparse +import json +import os +import platform as _stdlib_platform +import shlex # noqa: F401 (kept for potential future POSIX shell quoting) +import shutil +import subprocess +import sys +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +MAKE_PY_VERSION = "0.1.0" + +# --------------------------------------------------------------------------- +# Versions +# --------------------------------------------------------------------------- + + +@dataclass +class Versions: + """Parsed view of .devcontainer/versions.env. + + The format rules (documented in .devcontainer/README.md): + - One KEY=VALUE per line, no quotes, no spaces around `=`. + - Comments start with `#` at column 0. + - Blank lines are ignored. + - No shell expansion. + """ + + raw: dict[str, str] = field(default_factory=dict) + + @classmethod + def load(cls, repo_root: Path) -> "Versions": + path = repo_root / ".devcontainer" / "versions.env" + if not path.is_file(): + raise FileNotFoundError( + f"Missing {path}; cannot resolve pinned tool versions." + ) + raw: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + raw[k.strip()] = v.strip() + return cls(raw=raw) + + def get(self, key: str, default: Optional[str] = None) -> Optional[str]: + return self.raw.get(key, default) + + @property + def go_version(self) -> Optional[str]: + return self.raw.get("GO_VERSION") + + @property + def buf_version(self) -> Optional[str]: + return self.raw.get("BUF_VERSION") + + @property + def protobuf_version(self) -> Optional[str]: + """Resolved protobuf version for the active (DEFAULT_VARIANT) row.""" + return self._variant_value("PROTOBUF_VERSION") + + @property + def vcpkg_baseline_commit(self) -> Optional[str]: + """Resolved vcpkg baseline SHA for the active (DEFAULT_VARIANT) row.""" + return self._variant_value("VCPKG_BASELINE_COMMIT") + + @property + def default_variant(self) -> str: + """Active variant label (e.g. ``modern`` / ``legacy_v3``). + + Matched case-insensitively against variant-prefixed keys: a value of + ``modern`` resolves keys with prefix ``MODERN_``; ``legacy-v3`` / + ``legacy_v3`` both resolve ``LEGACY_V3_``. Defaults to ``modern`` if + the key is absent (preserves behaviour of older versions.env files). + """ + return (self.raw.get("DEFAULT_VARIANT") or "modern").strip().lower() + + def variants(self) -> dict[str, dict[str, str]]: + """Enumerate every (protobuf, vcpkg-baseline) variant defined. + + Returns ``{variant_label: {"protobuf_version": ..., "vcpkg_baseline_commit": ...}}`` + keyed by lowercase label. Labels that are missing one of the two keys + are still reported with the key they have, so callers can detect a + half-defined variant. + """ + suffixes = { + "_PROTOBUF_VERSION": "protobuf_version", + "_VCPKG_BASELINE_COMMIT": "vcpkg_baseline_commit", + } + out: dict[str, dict[str, str]] = {} + for k, v in self.raw.items(): + for sfx, field_name in suffixes.items(): + if k.endswith(sfx): + label = k[: -len(sfx)].lower() + out.setdefault(label, {})[field_name] = v + break + return out + + def _variant_value(self, suffix: str) -> Optional[str]: + """Resolve ``_`` (uppercase). Falls back to + the unprefixed key for backward compat with old versions.env files + that used the flat ``PROTOBUF_VERSION`` / ``VCPKG_BASELINE_COMMIT``.""" + prefix = self.default_variant.upper().replace("-", "_") + return self.raw.get(f"{prefix}_{suffix}") or self.raw.get(suffix) + + @property + def dotnet_version(self) -> Optional[str]: + return self.raw.get("DOTNET_VERSION") + + @property + def cmake_version(self) -> Optional[str]: + return self.raw.get("CMAKE_VERSION") + + +# --------------------------------------------------------------------------- +# Platform +# --------------------------------------------------------------------------- + + +@dataclass +class Platform: + """Host OS / arch / devcontainer detection plus a few helpers. + + The two helpers worth highlighting: + + - cmake_toolchain_args(): returns the right `-DCMAKE_TOOLCHAIN_FILE=... + -DVCPKG_TARGET_TRIPLET=...` flags on Windows; returns [] inside the + devcontainer (CMAKE_PREFIX_PATH=/opt/vcpkg/active is preset by the + Dockerfile) and on macOS / Linux (system protobuf is on PATH). + + - windows_msvc_wrap(cmd): on Windows wraps the command in + `cmd /c " x64 && "` so MSVC env lives only in the + single child process. On all other OSes returns cmd unchanged. + """ + + sys_platform: str + machine: str + in_devcontainer: bool + vcpkg_root: Optional[Path] = None + vcvarsall_path: Optional[Path] = None + protoc_tools_dir: Optional[Path] = None + vcpkg_installed_dir: Optional[Path] = None # manifest mode only + + @classmethod + def detect(cls) -> "Platform": + sys_platform = sys.platform + machine = _stdlib_platform.machine().lower() + # Devcontainer signal: only the marker the .devcontainer/Dockerfile + # actually sets. /.dockerenv is created by Docker for EVERY container + # and is too broad — using it would silently no-op `setup` in any + # plain Docker container. + in_devcontainer = Path("/opt/vcpkg/active").exists() + return cls( + sys_platform=sys_platform, + machine=machine, + in_devcontainer=in_devcontainer, + ) + + @property + def is_windows(self) -> bool: + return self.sys_platform.startswith("win") + + @property + def is_macos(self) -> bool: + return self.sys_platform == "darwin" + + @property + def is_linux(self) -> bool: + return self.sys_platform.startswith("linux") + + @property + def vcpkg_triplet(self) -> str: + """Default vcpkg triplet for the host. Mirrors the Dockerfile lines 32-36.""" + if self.is_windows: + return "x64-windows-static" + if self.is_macos: + if self.machine in ("arm64", "aarch64"): + return "arm64-osx" + return "x64-osx" + if self.is_linux: + if self.machine in ("arm64", "aarch64"): + return "arm64-linux" + return "x64-linux" + return "x64-linux" + + def cmake_toolchain_args( + self, triplet: Optional[str] = None, force_vcpkg: bool = False + ) -> list[str]: + """Extra cmake -D flags to pick up vcpkg's protobuf, when applicable. + + Default behaviour: + - devcontainer: [] (Dockerfile presets CMAKE_PREFIX_PATH). + - any host with VCPKG_ROOT: toolchain + triplet flags. + - host without vcpkg installed (Linux/macOS, no setup run yet): + [] (cmake will fall back to system + protobuf via find_package). + + force_vcpkg=True forces the toolchain flags even inside the + devcontainer (used by manifest-mode invocations that point cmake + at a custom vcpkg_installed/ dir). + """ + if self.in_devcontainer and not force_vcpkg: + return [] # Dockerfile presets CMAKE_PREFIX_PATH=/opt/vcpkg/active. + # Locate VCPKG_ROOT (cached attr or env). If absent on macOS/Linux, + # we silently fall through to system protobuf (apt/brew). On Windows + # / manifest mode the cpp handler errors with an actionable message + # before we get here. + vcpkg_root = self.vcpkg_root or _env_path("VCPKG_ROOT") + if vcpkg_root is None: + return [] + toolchain = vcpkg_root / "scripts" / "buildsystems" / "vcpkg.cmake" + return [ + f"-DCMAKE_TOOLCHAIN_FILE={toolchain}", + f"-DVCPKG_TARGET_TRIPLET={triplet or self.vcpkg_triplet}", + ] + + def windows_msvc_wrap(self, cmd: list[str]) -> list[str]: + """Wrap a command so it runs inside an MSVC-environment subshell. + + On Windows, returns a single-element list containing one cmd-shell + command string of the form: + 'call "" x64 >nul && ' + + Runner.run() detects the [windows-shell-string] pattern (via the + first element having no path separators but containing spaces) and + passes it to subprocess with shell=True so cmd parses it natively + — bypassing Python's Win32 CreateProcess quoting that otherwise + backslash-escapes the inner double quotes and breaks cmd's parser. + + On every other OS this returns `cmd` unchanged. + """ + if not self.is_windows: + return cmd + vcvars = self.vcvarsall_path or locate_vcvarsall() + if vcvars is None: + # No MSVC available — return cmd unchanged so the caller's + # subprocess fails with a useful "cl.exe not found" error rather + # than a more confusing wrapping failure. + return cmd + self.vcvarsall_path = vcvars + inner = " ".join(_winquote(arg) for arg in cmd) + # Single string: `call "" x64 >nul && `. + # `call` is required so vcvarsall returns control to our `&&`. + # `>nul` swallows vcvarsall's banner (cosmetic). + line = f'call "{vcvars}" x64 >nul && {inner}' + return [_WIN_SHELL_MARKER + line] + + +# Sentinel prefix used to flag a Runner.run() argv as "single shell string, +# please run via cmd". Using a prefix instead of a separate kwarg keeps +# windows_msvc_wrap() composable with the rest of the runner pipeline. +_WIN_SHELL_MARKER = "\x00CMDSHELL\x00" + + +def _env_path(name: str) -> Optional[Path]: + v = os.environ.get(name) + return Path(v) if v else None + + +def _winquote(arg: str) -> str: + """Quote an argument for the Windows cmd shell. + + cmd's quoting is famously bad. Wrap in double quotes if there is any + whitespace, ampersand, or other shell metacharacter. + """ + if not arg: + return '""' + if any(c in arg for c in ' &|<>^"()'): + # Escape embedded quotes by doubling them (cmd convention). + return '"' + arg.replace('"', '""') + '"' + return arg + + +def locate_vcvarsall() -> Optional[Path]: + """Find vcvarsall.bat on a Windows host. + + Strategy: + 1. Try vswhere.exe (the canonical method since VS 2017). + 2. Fall back to a few well-known install paths. + Returns None if MSVC isn't installed. + """ + if sys.platform != "win32": + return None + pf86 = os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)") + pf = os.environ.get("ProgramFiles", r"C:\Program Files") + for base in (pf86, pf): + vswhere = Path(base) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" + if vswhere.is_file(): + try: + out = subprocess.run( + [ + str(vswhere), + "-latest", + "-products", + "*", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "installationPath", + ], + capture_output=True, + text=True, + check=False, + ) + inst = out.stdout.strip().splitlines() + if inst: + candidate = ( + Path(inst[0]) / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat" + ) + if candidate.is_file(): + return candidate + except OSError: + pass + # Hardcoded fallbacks for VS 2022 Build Tools / Community / Enterprise. + for base in (pf86, pf): + for edition in ("BuildTools", "Community", "Professional", "Enterprise"): + candidate = ( + Path(base) + / "Microsoft Visual Studio" + / "2022" + / edition + / "VC" + / "Auxiliary" + / "Build" + / "vcvarsall.bat" + ) + if candidate.is_file(): + return candidate + return None + + +# --------------------------------------------------------------------------- +# Runner — subprocess helper +# --------------------------------------------------------------------------- + + +@dataclass +class Runner: + """Thin wrapper around subprocess.run with verbose / dry-run support.""" + + verbose: bool = False + dry_run: bool = False + + def run( + self, + cmd: list[str], + cwd: Optional[Path] = None, + env: Optional[dict[str, str]] = None, + check: bool = True, + shell: bool = False, + ) -> int: + # Detect the windows_msvc_wrap sentinel: a single-element argv whose + # value starts with _WIN_SHELL_MARKER. Strip the marker and run via + # cmd's native parser (shell=True) so Python's Win32 CreateProcess + # quoting doesn't mangle the embedded double quotes. + if ( + len(cmd) == 1 + and isinstance(cmd[0], str) + and cmd[0].startswith(_WIN_SHELL_MARKER) + ): + line = cmd[0][len(_WIN_SHELL_MARKER) :] + location = f" (cwd={cwd})" if cwd else "" + if self.dry_run: + print(f"[dry-run] {line}{location}") + return 0 + if self.verbose: + print(f"[run-shell] {line}{location}") + proc = subprocess.run( + line, + cwd=str(cwd) if cwd else None, + env=env, + check=False, + shell=True, + ) + if check and proc.returncode != 0: + raise SystemExit( + f"[error] command failed with exit code {proc.returncode}: {line}" + ) + return proc.returncode + + printable = " ".join(_winquote(c) if " " in c else c for c in cmd) + location = f" (cwd={cwd})" if cwd else "" + if self.dry_run: + print(f"[dry-run] {printable}{location}") + return 0 + if self.verbose: + print(f"[run] {printable}{location}") + proc = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + env=env, + check=False, + shell=shell, + ) + if check and proc.returncode != 0: + raise SystemExit( + f"[error] command failed with exit code {proc.returncode}: {printable}" + ) + return proc.returncode + + def rmtree(self, path: Path) -> None: + if self.dry_run: + # Always announce the wipe in dry-run, even if path is missing — + # it's the orchestrator's *intent* we want to capture. + print(f"[dry-run] rm -rf {path}") + return + if not path.exists(): + return + if self.verbose: + print(f"[rm-rf] {path}") + shutil.rmtree(path, ignore_errors=True) + + def mkdirp(self, path: Path) -> None: + if self.dry_run: + print(f"[dry-run] mkdir -p {path}") + return + if path.exists(): + return + if self.verbose: + print(f"[mkdir-p] {path}") + path.mkdir(parents=True, exist_ok=True) + + +# --------------------------------------------------------------------------- +# Repo root discovery +# --------------------------------------------------------------------------- + + +def find_repo_root(start: Optional[Path] = None) -> Path: + """Locate the repo root by walking upward to find .devcontainer/versions.env.""" + here = (start or Path(__file__).resolve()).parent if start is None else start + here = here.resolve() + for candidate in [here, *here.parents]: + if (candidate / ".devcontainer" / "versions.env").is_file(): + return candidate + raise SystemExit( + "[error] Could not locate repo root (no .devcontainer/versions.env found)." + ) + + +# --------------------------------------------------------------------------- +# ~/.loader-env.json — Windows toolchain cache +# --------------------------------------------------------------------------- + + +LOADER_ENV_PATH = Path.home() / ".loader-env.json" + + +def load_loader_env() -> dict: + if not LOADER_ENV_PATH.is_file(): + return {} + try: + return json.loads(LOADER_ENV_PATH.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + + +def save_loader_env(data: dict, runner: Runner) -> None: + text = json.dumps(data, indent=2, sort_keys=True) + if runner.dry_run: + print(f"[dry-run] write {LOADER_ENV_PATH}: {text}") + return + LOADER_ENV_PATH.write_text(text, encoding="utf-8") + + +# --------------------------------------------------------------------------- +# vcpkg baseline lookup +# --------------------------------------------------------------------------- +# +# Pinning protobuf via `overrides` while leaving `builtin-baseline` on a newer +# vcpkg commit lets transitive deps (abseil/utf8-range/re2/...) drift forward +# and breaks ABI (e.g. missing `absl::if_constexpr`). So for protobuf versions +# vcpkg's baseline.json knows about (>= 3.14.0), we pin the baseline itself to +# the vcpkg commit whose `versions/baseline.json` had our target protobuf — +# that whole snapshot is by construction self-consistent. +# +# Resolution: `git log -S '"X.Y.Z"' -- versions/baseline.json` → for each hit, +# verify `default.protobuf.baseline == X.Y.Z` at that commit (guards against +# the literal appearing in unrelated ports). Results cached in +# ~/.loader-env.json under "vcpkg_baseline_for_protobuf". +# +# Exception — protobuf < 3.14.0 (predates baseline.json, so no snapshot can +# carry it): we pin `builtin-baseline` at the *floor* (3.14.0) commit and add +# an `overrides` entry for the older protobuf. The ABI-drift hazard above does +# NOT apply here because pre-3.14 protobuf is dependency-light (no abseil / +# utf8-range / re2), and 3.14.0 is the oldest baseline available, so its +# transitive deps are already as old as vcpkg knows about. See +# `_resolve_vcpkg_protobuf_pin`. + + +def _resolve_vcpkg_baseline_for_protobuf( + protobuf_version: str, + vcpkg_root: Path, + runner: Runner, +) -> str: + """Find the vcpkg commit whose baseline.json has protobuf == protobuf_version. + + Raises RuntimeError if no such commit can be found in the local vcpkg + checkout — caller should surface a clear error and suggest `git fetch` + or a different --protobuf-version. + """ + # Cache hit? + cache = load_loader_env() + cached_map = cache.get("vcpkg_baseline_for_protobuf", {}) or {} + cached = cached_map.get(protobuf_version) + if cached: + # Validate the cached commit still has the expected protobuf — the + # user might have re-cloned vcpkg or rewritten history. + if _vcpkg_baseline_has_protobuf(vcpkg_root, cached, protobuf_version): + return cached + # Cache is stale; fall through to re-resolve. + cached_map.pop(protobuf_version, None) + + if runner.dry_run: + # Dry-run: don't shell out to git, return a placeholder so snapshot + # tests can still verify the command sequence. + return f"" + + if not (vcpkg_root / ".git").exists(): + raise RuntimeError( + f"{vcpkg_root} is not a git checkout; cannot resolve a vcpkg " + f"baseline commit for protobuf {protobuf_version}." + ) + + # Step 1: candidate commits via pickaxe search on the literal version string. + try: + out = subprocess.check_output( + [ + "git", + "-C", + str(vcpkg_root), + "log", + "-S", + f'"{protobuf_version}"', + "--reverse", + "--format=%H", + "--", + "versions/baseline.json", + ], + text=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"git log failed while searching vcpkg history for protobuf " + f"{protobuf_version}: {e}" + ) from e + + candidates = [line.strip() for line in out.splitlines() if line.strip()] + if not candidates: + raise RuntimeError( + f"No commit in {vcpkg_root} touches versions/baseline.json with " + f'the literal "{protobuf_version}". Possible causes:\n' + f" - your local vcpkg checkout is shallow / out-of-date " + f"(`git -C {vcpkg_root} fetch --unshallow origin master` may help)\n" + f" - protobuf {protobuf_version} was never the vcpkg baseline " + f"(use --vcpkg-baseline= to point at a custom snapshot)" + f"{_format_known_protobuf_versions_hint(vcpkg_root)}" + ) + + # Step 2: validate each candidate by reading baseline.json at that commit. + for sha in candidates: + if _vcpkg_baseline_has_protobuf(vcpkg_root, sha, protobuf_version): + cached_map[protobuf_version] = sha + cache["vcpkg_baseline_for_protobuf"] = cached_map + save_loader_env(cache, runner) + print( + f"[info] resolved vcpkg baseline for protobuf " + f"{protobuf_version} -> {sha[:12]}", + file=sys.stderr, + ) + return sha + + raise RuntimeError( + f'Found {len(candidates)} commits mentioning "{protobuf_version}" in ' + f"versions/baseline.json but none had default.protobuf.baseline set " + f"to that exact version (the literal likely appears in unrelated " + f"ports). Pass --vcpkg-baseline= manually." + f"{_format_known_protobuf_versions_hint(vcpkg_root)}" + ) + + +# Earliest protobuf version usable as a vcpkg `builtin-baseline`. +# vcpkg's versions/baseline.json was introduced 2021-01-21, and its very +# first commit already pinned `default.protobuf.baseline = "3.14.0"`. Port +# versions older than this (3.0.2, 3.2.0, ..., 3.13.0) exist in +# versions/p-/protobuf.json but were never the global baseline and so can't +# serve as builtin-baseline. The user-facing error message uses a rounder +# "~3.18" wording on purpose; the precise floor lives only here. +_VCPKG_PROTOBUF_BASELINE_FLOOR = (3, 14, 0) + +# Hard *support* floor — distinct from the vcpkg baseline floor above. Below +# 3.8.0 the C++ tableau-loader / generated code is known not to compile, even +# though vcpkg can still fetch those ancient ports via `overrides`. We refuse +# such versions up-front (before touching the user's trees) unless --force is +# given — a user may deliberately pick an unsupported version to investigate +# the compile breakage. Must stay <= _VCPKG_PROTOBUF_BASELINE_FLOOR so the +# below-baseline (overrides) range it gates is non-empty. +_PROTOBUF_MIN_SUPPORTED = (3, 8, 0) + + +def _parse_protobuf_version_tuple(v: str) -> Optional[tuple[int, ...]]: + """Parse "3.14.0" / "3.21.12" / "5.29.5" into a numeric tuple for ordering. + + Returns None for anything that isn't pure dotted-numeric (defensive: the + registry has historically been numeric-only for protobuf, but we'd + rather drop a weird entry than crash a hint formatter). + """ + parts = v.split(".") + if not all(p.isdigit() for p in parts) or not parts: + return None + return tuple(int(p) for p in parts) + + +def _ge_supported_floor(v: str) -> bool: + """True iff dotted version `v` parses and is >= the 3.8.0 support floor. + + Unparseable versions are treated as not-below-floor-worthy (returns False) + so they're dropped from below-floor listings rather than crashing. + """ + tup = _parse_protobuf_version_tuple(v) + return tup is not None and tup >= _PROTOBUF_MIN_SUPPORTED + + +def _read_protobuf_registry_versions(vcpkg_root: Path) -> list[str]: + """All protobuf versions in versions/p-/protobuf.json at HEAD. + + Reads the file with one cheap `git show` — vcpkg's authoritative list of + every protobuf version the registry has ever known. Returned in registry + order (newest first), de-duplicated, and *unfiltered* (includes pre-3.14 + versions). On any failure returns [] so callers can degrade gracefully. + """ + try: + blob = subprocess.check_output( + [ + "git", + "-C", + str(vcpkg_root), + "show", + "HEAD:versions/p-/protobuf.json", + ], + text=True, + stderr=subprocess.DEVNULL, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + return [] + try: + data = json.loads(blob) + except ValueError: + return [] + seen: set[str] = set() + versions: list[str] = [] + for entry in data.get("versions", []): + v = ( + entry.get("version") + or entry.get("version-semver") + or entry.get("version-string") + ) + if not v or v in seen: + continue + seen.add(v) + versions.append(v) + return versions + + +def _list_known_protobuf_versions(vcpkg_root: Path) -> list[str]: + """Return protobuf versions usable as a vcpkg `builtin-baseline`. + + Same source as `_read_protobuf_registry_versions`, filtered to + `>= _VCPKG_PROTOBUF_BASELINE_FLOOR` so the caller can blindly surface every + entry as something the user can pass to `--protobuf-version` and have pinned + *as the baseline* (older versions are still usable, but only via overrides). + """ + versions: list[str] = [] + for v in _read_protobuf_registry_versions(vcpkg_root): + tup = _parse_protobuf_version_tuple(v) + if tup is None or tup < _VCPKG_PROTOBUF_BASELINE_FLOOR: + continue + versions.append(v) + return versions + + +def _protobuf_version_in_registry(vcpkg_root: Path, version: str) -> bool: + """True iff `version` appears in versions/p-/protobuf.json at HEAD. + + This is the precondition for pinning it via `overrides`: vcpkg resolves an + override's version through the registry's version DB at the current + checkout, so a version absent here can never be installed. + """ + return version in set(_read_protobuf_registry_versions(vcpkg_root)) + + +def _format_known_protobuf_versions_hint( + vcpkg_root: Path, include_below_floor: bool = False +) -> str: + """Render a human-readable bullet listing known protobuf versions. + + Returns "" (caller can append unconditionally) when the version DB + can't be read. Otherwise returns a leading "\n - known versions: ..." + suitable for tacking onto the end of a RuntimeError message. When + `include_below_floor` is set, below-baseline versions are listed too — + down to the hard support floor (3.8.0) and *without* truncation — so a user + who asked for a missing older version can see every version they could pin + via `overrides`. Versions below the support floor are omitted (they won't + compile, so there's no point suggesting them). + """ + if include_below_floor: + versions = [ + v + for v in _read_protobuf_registry_versions(vcpkg_root) + if _ge_supported_floor(v) + ] + else: + versions = _list_known_protobuf_versions(vcpkg_root) + if not versions: + return "" + # Newest first (registry order). For the below-floor listing show the full + # range down to 3.8.0 (no truncation); for the baseline-only listing keep + # the compact head + "(+N older)" form. + if include_below_floor: + listed = ", ".join(versions) + else: + head = ", ".join(versions[:20]) + tail = f" (+{len(versions) - 20} older)" if len(versions) > 20 else "" + listed = f"{head}{tail}" + return ( + f"\n - known protobuf versions in {vcpkg_root}/versions/p-/protobuf.json " + f"(newest first, down to the {_PROTOBUF_MIN_SUPPORTED_STR} support " + f"floor): {listed}\n" + f" - note: versions older than 3.14.0 predate vcpkg's baseline.json, " + f"so they're pinned via `overrides` on the 3.14.0 baseline rather than " + f"as builtin-baseline (handled automatically by --protobuf-version)\n" + f" - note: {_PROTOBUF_MIN_SUPPORTED_STR} is the minimum supported " + f"version; anything older won't compile (pass --force to try anyway)" + ) + + +def _vcpkg_baseline_has_protobuf( + vcpkg_root: Path, commit_sha: str, expected_version: str +) -> bool: + """Read versions/baseline.json at commit_sha; return True iff protobuf matches.""" + try: + blob = subprocess.check_output( + [ + "git", + "-C", + str(vcpkg_root), + "show", + f"{commit_sha}:versions/baseline.json", + ], + text=True, + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + return False + try: + data = json.loads(blob) + except ValueError: + return False + entry = data.get("default", {}).get("protobuf", {}) + return entry.get("baseline") == expected_version + + +# The floor expressed as a dotted string ("3.14.0"), derived once from the +# tuple so the two never drift apart. This is the baseline we pin when a +# requested protobuf version predates vcpkg's baseline.json. +_VCPKG_PROTOBUF_BASELINE_FLOOR_STR = ".".join( + str(n) for n in _VCPKG_PROTOBUF_BASELINE_FLOOR +) + +# The hard support floor as a dotted string ("3.8.0"), derived from the tuple. +_PROTOBUF_MIN_SUPPORTED_STR = ".".join(str(n) for n in _PROTOBUF_MIN_SUPPORTED) + + +def _check_protobuf_min_supported(protobuf_version: str, force: bool) -> None: + """Guard against protobuf versions below the hard support floor (3.8.0). + + Below this floor the C++ loader / generated code is known not to compile. + Purely a version-number check (no vcpkg checkout needed), so it runs first + — before any baseline resolution or tree wipe. + + - version >= floor (or unparseable): no-op. + - version < floor, force=False: raise RuntimeError (caller fails fast). + - version < floor, force=True: warn loudly and continue, so the user + can reproduce / investigate the break. + """ + tup = _parse_protobuf_version_tuple(protobuf_version) + if tup is None or tup >= _PROTOBUF_MIN_SUPPORTED: + return + if force: + print( + f"[warn] protobuf {protobuf_version} is below the supported floor " + f"{_PROTOBUF_MIN_SUPPORTED_STR} and is NOT expected to compile; " + f"proceeding anyway because --force was given", + file=sys.stderr, + ) + return + raise RuntimeError( + f"protobuf {protobuf_version} is below the minimum supported version " + f"{_PROTOBUF_MIN_SUPPORTED_STR}; the C++ loader is not expected to " + f"compile against it, so the build is refused. Pass --force (-f) to " + f"attempt it anyway (e.g. to investigate the compile failure)." + ) + + +@dataclass +class VcpkgProtobufPin: + """How to pin protobuf into a generated vcpkg.json manifest. + + baseline: value for the manifest's ``builtin-baseline``. + override_version: when set, the requested protobuf predates vcpkg's + baseline.json, so it is pinned via an ``overrides`` + entry layered on top of the floor ``baseline``. ``None`` + means the baseline snapshot already carries the exact + protobuf version (no ``overrides`` needed). + """ + + baseline: str + override_version: Optional[str] = None + + +def _resolve_below_floor_override( + protobuf_version: str, + vcpkg_root: Path, + runner: Runner, +) -> Optional[str]: + """Decide whether `protobuf_version` must be pinned via `overrides`. + + Returns the version string when it predates the baseline.json floor (so the + caller should layer an `overrides` entry on the floor baseline), or None + when it is >= floor (pinned as the baseline itself). + + Raises RuntimeError when the version is below the floor *and* absent from + the vcpkg versions registry — letting callers fail fast (before wiping + codegen/build trees) instead of rendering a manifest that `vcpkg install` + would only reject minutes later. Skipped in dry-run, where there is no real + checkout to consult. + """ + tup = _parse_protobuf_version_tuple(protobuf_version) + if tup is None or tup >= _VCPKG_PROTOBUF_BASELINE_FLOOR: + return None + if not runner.dry_run and not _protobuf_version_in_registry( + vcpkg_root, protobuf_version + ): + raise RuntimeError( + f"protobuf {protobuf_version} predates vcpkg's baseline.json floor " + f"({_VCPKG_PROTOBUF_BASELINE_FLOOR_STR}) and is not present in " + f"{vcpkg_root}/versions/p-/protobuf.json, so it cannot be pinned via " + f"`overrides`. Possible causes:\n" + f" - typo in --protobuf-version\n" + f" - your local vcpkg checkout is shallow / out-of-date " + f"(`git -C {vcpkg_root} fetch origin master` may help)" + f"{_format_known_protobuf_versions_hint(vcpkg_root, include_below_floor=True)}" + ) + return protobuf_version + + +def _resolve_vcpkg_protobuf_pin( + protobuf_version: str, + vcpkg_root: Path, + runner: Runner, +) -> VcpkgProtobufPin: + """Resolve baseline (+ optional override) for a requested protobuf version. + + protobuf >= floor (3.14.0): pin the self-consistent baseline whose + ``versions/baseline.json`` carries that exact version — no ``overrides`` + (see module comment on transitive-dep ABI drift). + + protobuf < floor: pin the floor baseline and pin the older protobuf via + ``overrides``. Safe because pre-3.14 protobuf has no abseil/utf8-range/re2 + dependency, and the floor is the oldest baseline vcpkg has, so nothing + drifts forward. Raises RuntimeError early if the version doesn't exist. + """ + override = _resolve_below_floor_override(protobuf_version, vcpkg_root, runner) + if override is not None: + baseline = _resolve_vcpkg_baseline_for_protobuf( + _VCPKG_PROTOBUF_BASELINE_FLOOR_STR, vcpkg_root, runner + ) + print( + f"[info] protobuf {protobuf_version} predates vcpkg's baseline.json " + f"(floor {_VCPKG_PROTOBUF_BASELINE_FLOOR_STR}); pinning baseline " + f"{baseline[:12]} and adding overrides protobuf={protobuf_version}", + file=sys.stderr, + ) + return VcpkgProtobufPin(baseline=baseline, override_version=override) + baseline = _resolve_vcpkg_baseline_for_protobuf(protobuf_version, vcpkg_root, runner) + return VcpkgProtobufPin(baseline=baseline) + + +def hydrate_platform_from_env(plat: Platform) -> None: + """Populate Platform from $VCPKG_ROOT and ~/.loader-env.json. + + Cross-platform: macOS / Linux / Windows. Cache file is shared between + `make.py setup` (which writes it) and subsequent `make.py test` + invocations (which read it). + """ + if plat.vcpkg_root is None: + plat.vcpkg_root = _env_path("VCPKG_ROOT") + cache = load_loader_env() + if plat.vcpkg_root is None and cache.get("vcpkg_root"): + plat.vcpkg_root = Path(cache["vcpkg_root"]) + if plat.protoc_tools_dir is None and cache.get("protoc_tools_dir"): + plat.protoc_tools_dir = Path(cache["protoc_tools_dir"]) + if plat.vcpkg_installed_dir is None and cache.get("vcpkg_installed_dir"): + plat.vcpkg_installed_dir = Path(cache["vcpkg_installed_dir"]) + # vcvarsall is Windows-only. + if plat.is_windows: + if plat.vcvarsall_path is None and cache.get("vcvarsall_path"): + candidate = Path(cache["vcvarsall_path"]) + if candidate.is_file(): + plat.vcvarsall_path = candidate + + +# --------------------------------------------------------------------------- +# Setup commands +# --------------------------------------------------------------------------- + + +LANGS_ALL = ("go", "cpp", "csharp") + + +def _which(name: str) -> Optional[str]: + return shutil.which(name) + + +def _prepend_path(directory: Path) -> None: + """Prepend ``directory`` to ``os.environ['PATH']`` (idempotent, in-process). + + This makes a freshly-installed tool visible to (a) subsequent ``_which`` + probes inside the same ``setup`` run and (b) every child process spawned + by ``Runner`` afterwards (we never override ``env=``, so they inherit + ``os.environ``). No-op if the directory is already on PATH. + """ + p = str(directory) + sep = os.pathsep + current = os.environ.get("PATH", "") + parts = current.split(sep) if current else [] + # Case-insensitive comparison on Windows; exact match elsewhere. + norm = (lambda s: s.lower()) if os.name == "nt" else (lambda s: s) + if any(norm(x) == norm(p) for x in parts if x): + return + os.environ["PATH"] = p + (sep + current if current else "") + + +def cmd_setup(args, ctx: "Context") -> int: + """Install host toolchains. OS-dispatched. Idempotent.""" + if ctx.platform.in_devcontainer: + print("[info] Running inside devcontainer; toolchain already installed.") + return 0 + + langs = _resolve_langs(args.lang) + print(f"[info] Setting up host toolchain for: {', '.join(langs)}") + print(f"[info] Pinned versions: {ctx.versions.raw}") + + if ctx.platform.is_macos: + return _setup_macos(langs, ctx) + if ctx.platform.is_linux: + return _setup_linux(langs, ctx) + if ctx.platform.is_windows: + return _setup_windows(langs, ctx, skip_vcpkg=getattr(args, "skip_vcpkg", False)) + print( + f"[error] Unsupported host platform: {ctx.platform.sys_platform}", + file=sys.stderr, + ) + return 1 + + +def _resolve_langs(lang: str) -> list[str]: + if lang in (None, "all"): + return list(LANGS_ALL) + return [lang] + + +def _setup_macos(langs: list[str], ctx: "Context") -> int: + if _which("brew") is None: + print( + "[error] Homebrew is required on macOS. Install from https://brew.sh and re-run.", + file=sys.stderr, + ) + return 1 + cache = load_loader_env() + + # Brew packages: only the version-tolerant pieces (cmake, ninja, build + # essentials). Go and protobuf are pinned via tarball / vcpkg below; + # buf is pinned via direct download. dotnet@N is pinnable via brew's + # versioned formulae. + pkgs: list[str] = [] + if "cpp" in langs: + pkgs.extend(["cmake", "ninja"]) + if "csharp" in langs: + # `dotnet@8` (cask) covers .NET 8.x. Use the major. + major = (ctx.versions.dotnet_version or "8.0").split(".")[0] + pkgs.append(f"dotnet@{major}") + if pkgs: + ctx.runner.run(["brew", "update"], check=False) + ctx.runner.run(["brew", "install", *pkgs], check=False) + + # Pinned Go via official tarball (matches devcontainer). + if "go" in langs or "cpp" in langs or "csharp" in langs: + _ensure_go_tarball(ctx, "darwin") + + # Pinned buf via GitHub release. + _ensure_buf_unix(ctx, os_label="Darwin") + + # Pinned protobuf via vcpkg (matches devcontainer + Windows). + if "cpp" in langs: + _setup_vcpkg(ctx, cache) + + save_loader_env(cache, ctx.runner) + print("[info] macOS toolchain ready.") + return 0 + + +def _setup_linux(langs: list[str], ctx: "Context") -> int: + apt = _which("apt-get") is not None + dnf = _which("dnf") is not None + if not (apt or dnf): + print( + "[error] Neither apt-get nor dnf found. Install your toolchain manually.", + file=sys.stderr, + ) + return 1 + cache = load_loader_env() + + # Distro packages: only the version-tolerant pieces (cmake, ninja, build + # essentials, git). Go is pinned via tarball; buf via GitHub release; + # protobuf via vcpkg; .NET via Microsoft repo helper; Node via NodeSource. + base_pkgs: list[str] = [] + if "cpp" in langs: + if apt: + base_pkgs.extend( + ["cmake", "ninja-build", "build-essential", "git", "curl", "zip", "unzip", "tar", "pkg-config"] + ) + else: + base_pkgs.extend( + ["cmake", "ninja-build", "gcc-c++", "git", "curl", "zip", "unzip", "tar", "pkgconf-pkg-config"] + ) + + if base_pkgs: + sudo_prefix = ["sudo"] if os.geteuid() != 0 else [] + if apt: + ctx.runner.run([*sudo_prefix, "apt-get", "update"], check=False) + ctx.runner.run( + [*sudo_prefix, "apt-get", "install", "-y", *base_pkgs], check=False + ) + else: + ctx.runner.run( + [*sudo_prefix, "dnf", "install", "-y", *base_pkgs], check=False + ) + + # Pinned Go via official tarball. + if "go" in langs or "cpp" in langs or "csharp" in langs: + _ensure_go_tarball(ctx, "linux") + + # Pinned buf via GitHub release. + _ensure_buf_unix(ctx, os_label="Linux") + + if "csharp" in langs: + _ensure_dotnet_linux(ctx) + + # Pinned protobuf via vcpkg (matches devcontainer + Windows). + if "cpp" in langs: + _setup_vcpkg(ctx, cache) + + save_loader_env(cache, ctx.runner) + print("[info] Linux toolchain ready.") + return 0 + + +def _ensure_buf_unix(ctx: "Context", os_label: str) -> None: + """Download the pinned buf release to ~/.local/bin/buf. + + os_label: "Linux" or "Darwin" (matches the GitHub release naming). + + On a fresh host ``~/.local/bin`` is rarely on PATH (distros add it lazily + via ``~/.profile``, which a non-login shell never sources). After we drop + ``buf`` there we eagerly prepend the directory to ``os.environ['PATH']`` + so ``generate``/``test`` invoked later in the same ``setup`` run — and + every child process Runner spawns — can resolve ``buf`` without the user + having to relog or edit shell rc files. + """ + if _which("buf") is not None: + return + ver = ctx.versions.buf_version + if not ver: + return + target_dir = Path.home() / ".local" / "bin" + target = target_dir / "buf" + # Previously-installed binary that's just not on this shell's PATH — + # don't redownload, just expose it. + if target.exists(): + print(f"[info] buf already present at {target}; reusing.") + _prepend_path(target_dir) + return + arch = ( + "x86_64" + if _stdlib_platform.machine().lower() in ("x86_64", "amd64") + else "aarch64" + ) + url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-{os_label}-{arch}" + ctx.runner.mkdirp(target_dir) + print(f"[info] Downloading buf {ver} ({os_label}/{arch}) -> {target}") + if not ctx.runner.dry_run: + urllib.request.urlretrieve(url, str(target)) + target.chmod(0o755) + _prepend_path(target_dir) + + +def _ensure_dotnet_linux(ctx: "Context") -> None: + if _which("dotnet") is not None: + return + ver = ctx.versions.dotnet_version or "8.0" + script = Path.home() / ".local" / "bin" / "dotnet-install.sh" + ctx.runner.mkdirp(script.parent) + print(f"[info] Bootstrapping .NET {ver} via dotnet-install.sh") + if not ctx.runner.dry_run: + urllib.request.urlretrieve("https://dot.net/v1/dotnet-install.sh", str(script)) + script.chmod(0o755) + install_dir = Path.home() / ".dotnet" + ctx.runner.run( + [str(script), "--channel", ver, "--install-dir", str(install_dir)], + check=False, + ) + + +def _go_arch_macos() -> str: + return "arm64" if _stdlib_platform.machine().lower() in ("arm64", "aarch64") else "amd64" + + +def _go_arch_linux() -> str: + return "arm64" if _stdlib_platform.machine().lower() in ("arm64", "aarch64") else "amd64" + + +def _ensure_go_tarball(ctx: "Context", os_label: str) -> None: + """Install the official Go tarball at GO_VERSION into ~/.local/go/. + + Mirrors the devcontainer Dockerfile's Go layer. `go` already on PATH is + accepted as-is — bumping versions.env's GO_VERSION on a machine with an + older Go installed is the user's call (we don't auto-replace). + """ + if _which("go") is not None: + return + ver = ctx.versions.go_version + if not ver: + return + arch = _go_arch_macos() if os_label == "darwin" else _go_arch_linux() + url = f"https://go.dev/dl/go{ver}.{os_label}-{arch}.tar.gz" + target_root = Path.home() / ".local" + ctx.runner.mkdirp(target_root) + print(f"[info] Downloading Go {ver} ({os_label}/{arch}) -> {target_root}/go") + if ctx.runner.dry_run: + print(f"[dry-run] curl -L {url} | tar -C {target_root} -xz") + return + import tempfile + import tarfile + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp: + urllib.request.urlretrieve(url, tmp.name) + tarball = tmp.name + with tarfile.open(tarball, "r:gz") as tf: + tf.extractall(target_root) + os.unlink(tarball) + bin_dir = target_root / "go" / "bin" + print(f"[info] Go {ver} installed. Add to your shell profile:") + print(f" export PATH={bin_dir}:$PATH") + + +def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: + """Windows host setup.""" + cache = load_loader_env() + + # Step 0: Chocolatey + choco = _which("choco") + if choco is None: + print("[info] Chocolatey not found. Installing...") + ctx.runner.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + "[System.Net.ServicePointManager]::SecurityProtocol = " + "[System.Net.ServicePointManager]::SecurityProtocol -bor 3072; " + "iex ((New-Object System.Net.WebClient).DownloadString(" + "'https://community.chocolatey.org/install.ps1'))", + ], + check=False, + ) + else: + print(f"[info] Chocolatey already installed at: {choco}") + + if "cpp" in langs: + # Step 1: Ninja + if _which("ninja") is None: + print(f"[info] Installing ninja via choco...") + ctx.runner.run( + ["choco", "install", "ninja", "-y", "--no-progress"], check=False + ) + else: + print("[info] ninja already on PATH.") + + # Step 2: CMake + if _which("cmake") is None: + ver = ctx.versions.cmake_version or "3.31.8" + print(f"[info] Installing CMake {ver} via choco...") + ctx.runner.run( + [ + "choco", + "install", + "cmake", + f"--version={ver}", + "--installargs", + "ADD_CMAKE_TO_PATH=System", + "-y", + "--no-progress", + ], + check=False, + ) + else: + print("[info] cmake already on PATH.") + + # Step 3: MSVC Build Tools + vcvars = locate_vcvarsall() + if vcvars is None: + print( + "[info] MSVC Build Tools not found. Installing visualstudio2022buildtools..." + ) + ctx.runner.run( + [ + "choco", + "install", + "visualstudio2022buildtools", + "--package-parameters", + "--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --locale en-US", + "-y", + ], + check=False, + ) + vcvars = locate_vcvarsall() + if vcvars is not None: + print(f"[info] vcvarsall.bat: {vcvars}") + cache["vcvarsall_path"] = str(vcvars) + ctx.platform.vcvarsall_path = vcvars + + # Step 4: buf + if _which("buf") is None: + ver = ctx.versions.buf_version + if ver: + buf_dir = ( + Path(os.environ.get("LOCALAPPDATA", str(Path.home()))) / "buf" / "bin" + ) + buf_exe = buf_dir / "buf.exe" + if buf_exe.exists(): + # Previously-installed binary that just isn't on this shell's + # PATH — reuse it instead of redownloading. + print(f"[info] buf already present at {buf_exe}; reusing.") + _prepend_path(buf_dir) + else: + ctx.runner.mkdirp(buf_dir) + url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-Windows-x86_64.exe" + print(f"[info] Downloading buf {ver} -> {buf_exe}") + if not ctx.runner.dry_run: + urllib.request.urlretrieve(url, str(buf_exe)) + # Make buf usable for the rest of this `setup` run and any + # child process Runner spawns afterwards. Persisting the + # entry in the user's permanent PATH (registry / shell rc) + # is intentionally left to the user — modifying global env + # vars from a build script is too invasive. + _prepend_path(buf_dir) + print( + f"[info] buf installed at {buf_exe} (added to this session's PATH; " + f"add {buf_dir} to your user PATH to make it permanent)." + ) + else: + print("[info] buf already on PATH.") + + # Step 5: vcpkg + protobuf + if "cpp" in langs and not skip_vcpkg: + _setup_vcpkg(ctx, cache) + + # Step 6: Go (always required — every buf-generate invokes the Go + # protoc plugins via `go run ../../cmd/protoc-gen-...`, so Go must be + # on PATH regardless of which language is being built. Mirrors the + # macOS/Linux setup paths which install Go for any --lang.). + if _which("go") is None: + go_ver = ctx.versions.go_version or "1.24.0" + # winget package id format is `GoLang.Go..`; the patch + # level isn't part of the id (the package itself tracks it). + go_major_minor = ".".join(go_ver.split(".")[:2]) + ctx.runner.run( + ["winget", "install", "--id", f"GoLang.Go.{go_major_minor}", "-e"], + check=False, + ) + else: + print("[info] go already on PATH.") + + # Step 7: .NET SDK (only when csharp tests are requested). + if "csharp" in langs and _which("dotnet") is None: + dotnet_ver = ctx.versions.dotnet_version or "8.0" + # winget package id format is `Microsoft.DotNet.SDK.`. + dotnet_major = dotnet_ver.split(".")[0] + ctx.runner.run( + [ + "winget", + "install", + "--id", + f"Microsoft.DotNet.SDK.{dotnet_major}", + "-e", + ], + check=False, + ) + + save_loader_env(cache, ctx.runner) + print("[info] Windows toolchain ready.") + print( + "[info] Build/test commands run vcvarsall.bat per-process; your shell PATH is unchanged." + ) + return 0 + + +def _setup_vcpkg(ctx: "Context", cache: dict) -> None: + """Cross-platform vcpkg + protobuf installer (Windows / macOS / Linux). + + Mirrors the devcontainer Dockerfile's vcpkg layer so native dev gets the + same protobuf version pin as CI and the container. Idempotent — second + runs detect existing checkout and skip. + """ + triplet = ctx.platform.vcpkg_triplet + baseline = ctx.versions.vcpkg_baseline_commit + is_windows = ctx.platform.is_windows + bootstrap_name = "bootstrap-vcpkg.bat" if is_windows else "bootstrap-vcpkg.sh" + vcpkg_bin_name = "vcpkg.exe" if is_windows else "vcpkg" + protoc_bin_name = "protoc.exe" if is_windows else "protoc" + + # Resolve a usable vcpkg root: existing $VCPKG_ROOT, then cache, then a + # standard location under the user's home dir. + vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") + if vcpkg_root is None and cache.get("vcpkg_root"): + vcpkg_root = Path(cache["vcpkg_root"]) + + # Reject manifest-only vcpkg (no bootstrap script). On Windows this filters + # the VS-bundled vcpkg under `\VC\vcpkg\` which refuses classic mode. + if vcpkg_root is not None and not (vcpkg_root / bootstrap_name).is_file(): + print(f"[warn] {vcpkg_root} looks like a manifest-only vcpkg; ignoring.") + vcpkg_root = None + + home_default = ( + Path(os.environ.get("USERPROFILE", str(Path.home()))) / "vcpkg" + if is_windows + else Path.home() / "vcpkg" + ) + if vcpkg_root is None and (home_default / bootstrap_name).is_file(): + vcpkg_root = home_default + + if vcpkg_root is None: + vcpkg_root = home_default + print(f"[info] Cloning vcpkg into {vcpkg_root}...") + ctx.runner.run( + ["git", "clone", "https://github.com/microsoft/vcpkg.git", str(vcpkg_root)], + check=False, + ) + if baseline: + ctx.runner.run( + ["git", "-C", str(vcpkg_root), "fetch", "--quiet", "origin", baseline], + check=False, + ) + ctx.runner.run( + ["git", "-C", str(vcpkg_root), "checkout", "--quiet", baseline], + check=False, + ) + # Bootstrap. On Windows it shells out to MSVC's link.exe so wrap in + # vcvarsall; Unix bootstrap is self-contained (downloads prebuilt + # vcpkg binary or builds from source via cc). + bootstrap_path = vcpkg_root / bootstrap_name + bootstrap_cmd: list[str] = [str(bootstrap_path), "-disableMetrics"] + if is_windows: + ctx.runner.run( + ctx.platform.windows_msvc_wrap(bootstrap_cmd), + check=False, + ) + else: + # Make sure bootstrap.sh is executable (clone preserves permissions + # but a fresh git config sometimes drops the +x bit on Windows). + if not ctx.runner.dry_run and bootstrap_path.is_file(): + bootstrap_path.chmod(0o755) + ctx.runner.run(bootstrap_cmd, check=False) + + cache["vcpkg_root"] = str(vcpkg_root) + ctx.platform.vcpkg_root = vcpkg_root + + vcpkg_exe = vcpkg_root / vcpkg_bin_name + print(f"[info] vcpkg at: {vcpkg_root}") + + print(f"[info] Installing protobuf:{triplet} via vcpkg (classic mode)...") + install_cmd = [str(vcpkg_exe), "install", f"protobuf:{triplet}"] + if is_windows: + ctx.runner.run( + ctx.platform.windows_msvc_wrap(install_cmd), + check=False, + ) + else: + ctx.runner.run(install_cmd, check=False) + + protoc_dir = vcpkg_root / "installed" / triplet / "tools" / "protobuf" + if (protoc_dir / protoc_bin_name).is_file() or ctx.runner.dry_run: + cache["protoc_tools_dir"] = str(protoc_dir) + ctx.platform.protoc_tools_dir = protoc_dir + + +# --------------------------------------------------------------------------- +# generate / build / test / clean +# --------------------------------------------------------------------------- + + +def _lang_dir(repo_root: Path, lang: str) -> Path: + return repo_root / "test" / f"{lang}-tableau-loader" + + +def _buf_generate( + ctx: "Context", lang: str, protoc_dir_override: Optional[Path] = None +) -> None: + cwd = _lang_dir(ctx.repo_root, lang) + cmd = ctx.platform.windows_msvc_wrap(["buf", "generate", ".."]) + env = os.environ.copy() + # Pick the protoc to put on PATH: + # - protoc_dir_override (manifest-mode protoc from this build's + # vcpkg_installed/) wins — required when --protobuf-version is set, + # so codegen matches the libprotobuf headers cmake will use. + # - Else the classic-mode protoc cached in ~/.loader-env.json + # (populated on every host by `make.py setup`). + protoc_dir = protoc_dir_override or ctx.platform.protoc_tools_dir + if protoc_dir is not None: + env["PATH"] = f"{protoc_dir}{os.pathsep}{env.get('PATH', '')}" + ctx.runner.run(cmd, cwd=cwd, env=env) + + +def cmd_generate(args, ctx: "Context") -> int: + _buf_generate(ctx, args.lang) + return 0 + + +def cmd_build(args, ctx: "Context") -> int: + return _build_or_test(args, ctx, run_tests=False) + + +def cmd_test(args, ctx: "Context") -> int: + return _build_or_test(args, ctx, run_tests=True) + + +def _build_or_test(args, ctx: "Context", run_tests: bool) -> int: + lang = args.lang + if lang == "go": + return _go_build_or_test(args, ctx, run_tests) + if lang == "cpp": + return _cpp_build_or_test(args, ctx, run_tests) + if lang == "csharp": + return _csharp_build_or_test(args, ctx, run_tests) + print(f"[error] unknown --lang {lang}", file=sys.stderr) + return 2 + + +# ----- Go ----- + + +def _go_build_or_test(args, ctx: "Context", run_tests: bool) -> int: + cwd = _lang_dir(ctx.repo_root, "go") + + if run_tests and getattr(args, "smoke", False): + # Devcontainer-smoke equivalent — vet plugin packages only, skip ./test/... + # No buf-generate needed: those packages don't depend on freshly + # generated *.pb.go. + ctx.runner.run( + [ + "go", + "vet", + "./cmd/...", + "./pkg/...", + "./internal/options/...", + "./internal/loadutil/...", + "./internal/xproto/...", + ], + cwd=ctx.repo_root, + ) + return 0 + + # Always regenerate, mirroring CI. + if not getattr(args, "no_generate", False): + _buf_generate(ctx, "go") + + if not run_tests: + ctx.runner.run(["go", "build", "./..."], cwd=cwd) + return 0 + + cmd = ["go", "test", "-v", "-timeout", "30m"] + # Resolve --race default based on host OS. Windows requires cgo (and a + # C compiler) for -race; Linux/macOS work out of the box. + race = getattr(args, "race", None) + if race is None: + race = not ctx.platform.is_windows + if race: + cmd.append("-race") + if getattr(args, "coverage", False): + cmd.extend(["-coverprofile=coverage.txt", "-covermode=atomic"]) + cmd.append("./...") + if getattr(args, "k", None): + cmd.extend(["-run", args.k]) + ctx.runner.run(cmd, cwd=cwd) + return 0 + + +# ----- C++ ----- + + +def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: + cwd = _lang_dir(ctx.repo_root, "cpp") + triplet = getattr(args, "triplet", None) or ctx.platform.vcpkg_triplet + protobuf_version = getattr(args, "protobuf_version", None) + cxx_std = getattr(args, "cxx_std", "17") + cxx_compiler = getattr(args, "cxx_compiler", None) + + # Resolve the vcpkg baseline up-front when --protobuf-version is given. + # This MUST happen before the stale-codegen rmtree below: an invalid + # --protobuf-version that would later raise should not have already + # wiped the user's src/protoconf, src/tableau, build/ trees. + # + # Baseline resolution order: + # 1. --vcpkg-baseline= (explicit override) + # 2. _resolve_vcpkg_protobuf_pin(...) (auto: git-search vcpkg) + # For protobuf < 3.14.0 an `overrides` entry is layered on top of the + # floor baseline (see _resolve_vcpkg_protobuf_pin / module comment). + vcpkg_root: Optional[Path] = None + baseline: Optional[str] = None + override_version: Optional[str] = None + if protobuf_version: + # Hard support-floor guard first — purely a version-number check (no + # vcpkg checkout needed), so it runs before baseline resolution and the + # tree wipe below. Refuses protobuf < 3.8.0 unless --force is given. + try: + _check_protobuf_min_supported( + protobuf_version, getattr(args, "force", False) + ) + except RuntimeError as e: + print(f"[error] {e}", file=sys.stderr) + return 1 + # Need a vcpkg checkout to resolve the baseline by git history. + vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") + if vcpkg_root is None: + if ctx.runner.dry_run: + # Dry-run: print what would happen with a placeholder so the + # snapshot test can still verify the command sequence. + vcpkg_root = Path("") + else: + print( + "[error] --protobuf-version requires VCPKG_ROOT to be set " + "(run `python3 make.py setup --lang cpp` first, or set " + "VCPKG_ROOT in your environment).", + file=sys.stderr, + ) + return 1 + # Surface the resolved root on the platform so cmake_toolchain_args + # picks it up for the configure command. + ctx.platform.vcpkg_root = vcpkg_root + + explicit_baseline = getattr(args, "vcpkg_baseline", None) + try: + if explicit_baseline: + baseline = explicit_baseline + print( + f"[info] using --vcpkg-baseline={baseline[:12]}", + file=sys.stderr, + ) + # A version below the baseline.json floor can't be carried by + # any baseline snapshot, so even with an explicit baseline we + # must pin it via overrides. _resolve_below_floor_override also + # validates the version exists (raises early otherwise). + override_version = _resolve_below_floor_override( + protobuf_version, vcpkg_root, ctx.runner + ) + if override_version: + print( + f"[info] protobuf {protobuf_version} predates baseline.json " + f"floor {_VCPKG_PROTOBUF_BASELINE_FLOOR_STR}; adding " + f"overrides protobuf={protobuf_version}", + file=sys.stderr, + ) + else: + pin = _resolve_vcpkg_protobuf_pin( + protobuf_version, vcpkg_root, ctx.runner + ) + baseline = pin.baseline + override_version = pin.override_version + except RuntimeError as e: + # Fail fast — user trees on disk are still untouched. + print(f"[error] {e}", file=sys.stderr) + print( + "[hint] If you know a vcpkg commit whose baseline matches " + "your protobuf, pass --vcpkg-baseline= to skip auto-resolution.", + file=sys.stderr, + ) + return 1 + + # Stale-codegen wipe (gitignored *.pb.* files left over from a previous + # protoc version shadow fresh codegen). Skip with --no-clean. + if not getattr(args, "no_clean", False): + ctx.runner.rmtree(cwd / "build") + ctx.runner.rmtree(cwd / "src" / "tableau") + ctx.runner.rmtree(cwd / "src" / "protoconf") + + # Classic mode: a stale vcpkg.json from a previous --protobuf-version run + # would silently switch cmake's vcpkg toolchain into manifest mode and + # build the wrong libprotobuf into build/vcpkg_installed/. Always remove + # it here unless we're about to render a fresh one below. + if not protobuf_version: + manifest_path = cwd / "vcpkg.json" + if manifest_path.is_file(): + if ctx.runner.dry_run: + print(f"[dry-run] rm {manifest_path}") + else: + manifest_path.unlink() + + # Manifest mode: render vcpkg.json pinning the requested protobuf-version, + # then run `vcpkg install` to populate vcpkg_installed/. This matches CI's + # testing-cpp.yml flow (which uses lukka/run-vcpkg with runVcpkgInstall: + # true) and means switching --protobuf-version Just Works without + # re-running `make.py setup`. Idempotent: vcpkg detects already-installed + # packages and skips them. + cmake_extra: list[str] = [] + if protobuf_version: + # vcpkg_root and baseline were resolved above; assert for the type + # checker (both are guaranteed non-None inside this branch). + assert vcpkg_root is not None and baseline is not None + + manifest = { + "name": "loader-cpp-test", + "version": "0.1.0", + "dependencies": ["protobuf"], + "builtin-baseline": baseline, + } + # Below-floor protobuf: pin it via overrides on top of the floor + # baseline (resolved in _resolve_vcpkg_protobuf_pin). + if override_version: + manifest["overrides"] = [ + {"name": "protobuf", "version": override_version} + ] + manifest_path = cwd / "vcpkg.json" + if not ctx.runner.dry_run: + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + installed_dir = Path( + os.environ.get("VCPKG_INSTALLED_DIR", str(cwd / "vcpkg_installed")) + ) + + vcpkg_exe = vcpkg_root / ("vcpkg.exe" if ctx.platform.is_windows else "vcpkg") + + # Install the manifest: must `cd` into the manifest dir for vcpkg to + # discover it. Skip when --no-vcpkg-install is passed (CI's + # lukka/run-vcpkg already did it). + if not getattr(args, "no_vcpkg_install", False): + install_cmd = [ + str(vcpkg_exe), + "install", + f"--triplet={triplet}", + f"--x-install-root={installed_dir}", + ] + ctx.runner.run( + ctx.platform.windows_msvc_wrap(install_cmd), + cwd=cwd, + ) + + cmake_extra.extend( + [ + f"-DVCPKG_INSTALLED_DIR={installed_dir}", + "-DVCPKG_MANIFEST_INSTALL=OFF", + ] + ) + + if not getattr(args, "no_generate", False): + # In manifest mode, route buf-generate's protoc to the manifest's + # tools dir so codegen matches the libprotobuf cmake links against. + protoc_dir_override: Optional[Path] = None + if protobuf_version: + protoc_dir_override = installed_dir / triplet / "tools" / "protobuf" + _buf_generate(ctx, "cpp", protoc_dir_override=protoc_dir_override) + + configure_cmd = [ + "cmake", + "-S", + ".", + "-B", + "build", + "-DCMAKE_BUILD_TYPE=Debug", + f"-DCMAKE_CXX_STANDARD={cxx_std}", + ] + if cxx_compiler: + # Translate friendly names. + compiler = {"msvc": "cl", "clang": "clang++", "gcc": "g++"}.get( + cxx_compiler, cxx_compiler + ) + configure_cmd.append(f"-DCMAKE_CXX_COMPILER={compiler}") + if shutil.which("ninja") is not None: + configure_cmd.extend(["-G", "Ninja"]) + configure_cmd.extend( + ctx.platform.cmake_toolchain_args( + triplet=triplet, + # Manifest mode means we ARE using vcpkg regardless of host; + # force the toolchain flags even on Linux/macOS so cmake's + # find_package(Protobuf) resolves against vcpkg_installed/. + force_vcpkg=bool(protobuf_version), + ) + ) + configure_cmd.extend(cmake_extra) + + ctx.runner.run(ctx.platform.windows_msvc_wrap(configure_cmd), cwd=cwd) + ctx.runner.run( + ctx.platform.windows_msvc_wrap(["cmake", "--build", "build", "--parallel"]), + cwd=cwd, + ) + + if run_tests: + ctest_cmd = ["ctest", "--test-dir", "build", "--output-on-failure"] + if getattr(args, "k", None): + ctest_cmd.extend(["-R", args.k]) + ctx.runner.run(ctx.platform.windows_msvc_wrap(ctest_cmd), cwd=cwd) + return 0 + + +# ----- C# ----- + + +def _csharp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: + cwd = _lang_dir(ctx.repo_root, "csharp") + if not getattr(args, "no_generate", False): + _buf_generate(ctx, "csharp") + + if not run_tests: + ctx.runner.run(["dotnet", "build", "--nologo"], cwd=cwd) + return 0 + + cmd = ["dotnet", "test", "--nologo", "--logger", "console;verbosity=normal"] + if getattr(args, "k", None): + cmd.extend(["--filter", f"FullyQualifiedName~{args.k}"]) + ctx.runner.run(cmd, cwd=cwd) + return 0 + + +# ----- clean / env ----- + + +def cmd_clean(args, ctx: "Context") -> int: + targets: list[str] = [] + if args.all: + targets = list(LANGS_ALL) + else: + targets = _resolve_langs(args.lang) + for lang in targets: + cwd = _lang_dir(ctx.repo_root, lang) + if lang == "cpp": + ctx.runner.rmtree(cwd / "build") + ctx.runner.rmtree(cwd / "src" / "tableau") + ctx.runner.rmtree(cwd / "src" / "protoconf") + elif lang == "csharp": + ctx.runner.rmtree(cwd / "bin") + ctx.runner.rmtree(cwd / "obj") + ctx.runner.rmtree(cwd / "protoconf") + elif lang == "go": + ctx.runner.rmtree(cwd / "protoconf") + return 0 + + +def cmd_env(args, ctx: "Context") -> int: + info = { + "make_py_version": MAKE_PY_VERSION, + "repo_root": str(ctx.repo_root), + "sys_platform": ctx.platform.sys_platform, + "machine": ctx.platform.machine, + "in_devcontainer": ctx.platform.in_devcontainer, + "vcpkg_triplet": ctx.platform.vcpkg_triplet, + "vcpkg_root": str(ctx.platform.vcpkg_root) if ctx.platform.vcpkg_root else None, + "vcvarsall_path": ( + str(ctx.platform.vcvarsall_path) if ctx.platform.vcvarsall_path else None + ), + "protoc_tools_dir": ( + str(ctx.platform.protoc_tools_dir) + if ctx.platform.protoc_tools_dir + else None + ), + "tools": { + "go": _which("go"), + "buf": _which("buf"), + "protoc": _which("protoc"), + "cmake": _which("cmake"), + "ninja": _which("ninja"), + "dotnet": _which("dotnet"), + }, + "versions_env": ctx.versions.raw, + } + print(json.dumps(info, indent=2)) + return 0 + + +# --------------------------------------------------------------------------- +# Context + arg parsing + main +# --------------------------------------------------------------------------- + + +@dataclass +class Context: + repo_root: Path + versions: Versions + platform: Platform + runner: Runner + + +def build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="make.py", + description="Cross-platform build/test driver for tableauio/loader.", + ) + p.add_argument( + "--version", action="store_true", help="print make.py version + versions.env" + ) + p.add_argument("-v", "--verbose", action="store_true") + p.add_argument("--dry-run", action="store_true") + p.add_argument("--cwd", type=str, default=None, help="repo root override") + + sub = p.add_subparsers(dest="command") + + # setup + sp = sub.add_parser("setup", help="Install host toolchain") + sp.add_argument("--lang", choices=[*LANGS_ALL, "all"], default="all") + sp.add_argument( + "--skip-vcpkg", + action="store_true", + help="(Windows) skip vcpkg install (CI uses lukka/run-vcpkg)", + ) + + # generate + sp = sub.add_parser("generate", help="Run buf generate for a language") + sp.add_argument("--lang", choices=LANGS_ALL, required=True) + + # build + sp = sub.add_parser("build", help="Compile generated code for a language") + _add_build_flags(sp) + + # test + sp = sub.add_parser("test", help="Run tests for a language") + _add_build_flags(sp) + sp.add_argument( + "-k", + type=str, + default=None, + help="test filter (go -run / ctest -R / dotnet --filter)", + ) + sp.add_argument( + "--smoke", action="store_true", help="(go only) smoke vet, no full test run" + ) + # --race / --no-race: tri-state. Default depends on host OS: + # Linux/macOS -> default ON (-race works out of the box). + # Windows -> default OFF (-race needs cgo+C compiler; users opt in + # explicitly via --race once they have MSVC/MinGW). + sp.add_argument( + "--race", + dest="race", + action="store_true", + default=None, + help="enable -race (default on Linux/macOS, off on Windows)", + ) + sp.add_argument( + "--no-race", dest="race", action="store_false", help="disable -race" + ) + sp.add_argument("--coverage", action="store_true") + + # clean + sp = sub.add_parser("clean", help="Wipe generated code + build outputs") + sp.add_argument("--lang", choices=[*LANGS_ALL, "all"], default="all") + sp.add_argument("--all", action="store_true") + + # env + sub.add_parser("env", help="Print resolved environment as JSON") + + return p + + +def _add_build_flags(sp: argparse.ArgumentParser) -> None: + sp.add_argument("--lang", choices=LANGS_ALL, required=True) + sp.add_argument("--cxx-std", choices=["17", "20"], default="17") + sp.add_argument("--cxx-compiler", choices=["msvc", "clang", "gcc"], default=None) + sp.add_argument( + "--protobuf-version", + type=str, + default=None, + help="(cpp) pin vcpkg protobuf port to this version (manifest mode)", + ) + sp.add_argument( + "--vcpkg-baseline", + type=str, + default=None, + help=( + "(cpp manifest mode) explicit vcpkg builtin-baseline commit; " + "skips auto-resolution from --protobuf-version" + ), + ) + sp.add_argument( + "--triplet", type=str, default=None, help="(cpp) vcpkg triplet override" + ) + sp.add_argument( + "-f", + "--force", + action="store_true", + help=( + "(cpp) attempt the build even with a protobuf version below the " + "3.8.0 support floor (expected to fail to compile; useful for " + "investigating the breakage)" + ), + ) + sp.add_argument( + "--no-clean", + action="store_true", + help="(cpp) skip pre-build wipe of build/ + generated codegen", + ) + sp.add_argument( + "--no-vcpkg-install", + action="store_true", + help="(cpp manifest mode) skip `vcpkg install` (CI uses lukka/run-vcpkg)", + ) + sp.add_argument( + "--no-generate", action="store_true", help="skip the buf-generate step" + ) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + + if args.version: + repo = ( + find_repo_root(Path(args.cwd).resolve()) if args.cwd else find_repo_root() + ) + v = Versions.load(repo) + print(f"make.py {MAKE_PY_VERSION}") + for k, val in v.raw.items(): + print(f" {k}={val}") + return 0 + + if args.command is None: + parser.print_help() + return 0 + + repo = find_repo_root(Path(args.cwd).resolve()) if args.cwd else find_repo_root() + versions = Versions.load(repo) + plat = Platform.detect() + hydrate_platform_from_env(plat) + runner = Runner(verbose=args.verbose, dry_run=args.dry_run) + ctx = Context(repo_root=repo, versions=versions, platform=plat, runner=runner) + + dispatch = { + "setup": cmd_setup, + "generate": cmd_generate, + "build": cmd_build, + "test": cmd_test, + "clean": cmd_clean, + "env": cmd_env, + } + handler = dispatch.get(args.command) + if handler is None: + parser.print_help() + return 2 + return handler(args, ctx) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/prepare.bat b/prepare.bat deleted file mode 100644 index 8f3f0231..00000000 --- a/prepare.bat +++ /dev/null @@ -1,291 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -REM ----------------------------------------------------------------------- -REM Parse arguments -REM --dry-run : print what would be done, but do not install anything -REM --simulate-clean : pretend nothing is installed (implies --dry-run) -REM ----------------------------------------------------------------------- -set "DRY_RUN=0" -set "SIMULATE_CLEAN=0" -for %%A in (%*) do ( - if /i "%%A"=="--dry-run" set "DRY_RUN=1" - if /i "%%A"=="--simulate-clean" set "DRY_RUN=1" & set "SIMULATE_CLEAN=1" -) -if "%DRY_RUN%"=="1" echo [DRY-RUN] No changes will be made to the system. -if "%SIMULATE_CLEAN%"=="1" echo [DRY-RUN] Simulating a clean machine (all tools treated as not installed). - -echo [INFO] Preparing build environment... - -REM ----------------------------------------------------------------------- -REM Step 0: Ensure Chocolatey is installed -REM ----------------------------------------------------------------------- -set "CHOCO_EXE=" -set "CHOCO_BASE=" -if "%SIMULATE_CLEAN%"=="0" ( - REM Try env var first, then fall back to registry (HKCU then HKLM) - if defined ChocolateyInstall set "CHOCO_BASE=%ChocolateyInstall%" - if not defined CHOCO_BASE ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" - ) - if not defined CHOCO_BASE ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" - ) - if not defined CHOCO_BASE set "CHOCO_BASE=%ALLUSERSPROFILE%\chocolatey" - if exist "!CHOCO_BASE!\bin\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\bin\choco.exe" - if exist "!CHOCO_BASE!\redirects\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\redirects\choco.exe" - if exist "!CHOCO_BASE!\tools\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\tools\choco.exe" -) -if not defined CHOCO_EXE ( - echo [INFO] Chocolatey not found. Installing Chocolatey... - if "%DRY_RUN%"=="0" ( - powershell -NoProfile -ExecutionPolicy Bypass -Command ^ - "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))" - if errorlevel 1 ( - echo [ERROR] Failed to install Chocolatey. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: powershell ... install Chocolatey - ) - REM Add Chocolatey to current session PATH - set "PATH=%ALLUSERSPROFILE%\chocolatey\bin;%PATH%" - REM Persist Chocolatey bin to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"%ALLUSERSPROFILE%\chocolatey\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "%ALLUSERSPROFILE%\chocolatey\bin;!USR_PATH!" - echo [INFO] Chocolatey bin added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "%%ALLUSERSPROFILE%%\chocolatey\bin;..." - ) - echo [INFO] Chocolatey installed successfully. -) else ( - echo [INFO] Chocolatey already installed. -) - -REM Refresh ChocolateyInstall var if it was just installed (also read from registry) -if not defined ChocolateyInstall ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "ChocolateyInstall=%%b" -) -if not defined ChocolateyInstall set "ChocolateyInstall=%ALLUSERSPROFILE%\chocolatey" -if "%SIMULATE_CLEAN%"=="0" ( - set "PATH=%ChocolateyInstall%\bin;%ChocolateyInstall%\lib\ninja\tools;%PATH%" -) - -REM ----------------------------------------------------------------------- -REM Step 1: Ensure Ninja is installed via Chocolatey -REM ----------------------------------------------------------------------- -set "NINJA_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where ninja.exe >nul 2>&1 - if not errorlevel 1 set "NINJA_FOUND=1" -) -if "%NINJA_FOUND%"=="0" ( - echo [INFO] ninja.exe not found. Installing via choco... - if "%DRY_RUN%"=="0" ( - choco install ninja -y --no-progress - if errorlevel 1 ( - echo [ERROR] Failed to install ninja. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: choco install ninja -y --no-progress - ) - REM Add ninja to current session PATH - if defined ChocolateyInstall ( - set "NINJA_PATH=!ChocolateyInstall!\lib\ninja\tools" - ) else ( - set "NINJA_PATH=%ALLUSERSPROFILE%\chocolatey\lib\ninja\tools" - ) - set "PATH=!NINJA_PATH!;%PATH%" - REM Persist ninja path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"ninja\tools" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!NINJA_PATH!;!USR_PATH!" - echo [INFO] ninja path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!NINJA_PATH!;..." - ) - echo [INFO] ninja installed successfully. -) else ( - echo [INFO] ninja.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 2: Ensure CMake 3.31.8 is installed -REM Try Chocolatey first; fall back to direct MSI download. -REM ----------------------------------------------------------------------- -set "CMAKE_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where cmake.exe >nul 2>&1 - if not errorlevel 1 set "CMAKE_FOUND=1" -) -if "%CMAKE_FOUND%"=="0" ( - echo [INFO] cmake.exe not found. Installing CMake 3.31.8... - if "%DRY_RUN%"=="0" ( - set "CMAKE_INSTALLED=0" - REM --- Attempt 1: Chocolatey --- - choco install cmake --version=3.31.8 --installargs "'ADD_CMAKE_TO_PATH=System'" -y --no-progress >nul 2>&1 && set "CMAKE_INSTALLED=1" - if "!CMAKE_INSTALLED!"=="0" ( - echo [WARN] choco install cmake failed. Falling back to direct MSI download... - set "CMAKE_MSI=%TEMP%\cmake-3.31.8-windows-x86_64.msi" - powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-windows-x86_64.msi','!CMAKE_MSI!')" - if not exist "!CMAKE_MSI!" ( - echo [ERROR] Failed to download CMake MSI. - exit /b 1 - ) - msiexec /i "!CMAKE_MSI!" ADD_CMAKE_TO_PATH=System /quiet /norestart - if errorlevel 1 ( - echo [ERROR] Failed to install CMake from MSI. - exit /b 1 - ) - del /q "!CMAKE_MSI!" 2>nul - ) - ) else ( - echo [DRY-RUN] Would run: choco install cmake --version=3.31.8 ... (or fallback to MSI download) - ) - REM Add cmake to current session PATH - set "CMAKE_PATH=C:\Program Files\CMake\bin" - set "PATH=!CMAKE_PATH!;%PATH%" - REM Persist cmake path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"CMake\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!CMAKE_PATH!;!USR_PATH!" - echo [INFO] cmake path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!CMAKE_PATH!;..." - ) - echo [INFO] cmake installed successfully. -) else ( - echo [INFO] cmake.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 3: Ensure MSVC compiler (cl.exe) is available, then activate its -REM environment for this cmd session via vcvarsall.bat. The CI -REM workflow uses ilammy/msvc-dev-cmd@v1 to do the same thing. -REM ----------------------------------------------------------------------- -set "CL_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where cl.exe >nul 2>&1 - if not errorlevel 1 set "CL_FOUND=1" -) -set "SKIP_MSVC=0" -if "%CL_FOUND%"=="0" ( - echo [INFO] cl.exe not found. Searching for existing VS installation... - set "VSWHERE=" - if "%SIMULATE_CLEAN%"=="0" ( - for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( - if not defined VSWHERE ( - if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( - set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" - ) - ) - ) - ) - if not defined VSWHERE ( - echo [INFO] Visual Studio not found. Installing via choco... - if "%DRY_RUN%"=="0" ( - choco install visualstudio2022buildtools --package-parameters "--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --locale en-US" -y - if errorlevel 1 ( - echo [ERROR] Failed to install Visual Studio Build Tools. - exit /b 1 - ) - echo [INFO] Visual Studio Build Tools installed successfully. - REM Re-search vswhere after installation - for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( - if not defined VSWHERE ( - if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( - set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" - ) - ) - ) - ) else ( - echo [DRY-RUN] Would run: choco install visualstudio2022buildtools ... - echo [DRY-RUN] Would search vswhere.exe after installation. - set "SKIP_MSVC=1" - ) - ) - if "!SKIP_MSVC!"=="0" ( - if not defined VSWHERE ( - echo [ERROR] vswhere.exe still not found after installation. Please restart and retry. - exit /b 1 - ) - set "VCVARSALL=" - for /f "usebackq delims=" %%p in (`"!VSWHERE!" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( - set "VCVARSALL=%%p\VC\Auxiliary\Build\vcvarsall.bat" - ) - if not defined VCVARSALL ( - echo [ERROR] No VS installation with C++ tools detected. - exit /b 1 - ) - if not exist "!VCVARSALL!" ( - echo [ERROR] vcvarsall.bat not found at: !VCVARSALL! - exit /b 1 - ) - echo [INFO] Initializing MSVC environment from: !VCVARSALL! - call "!VCVARSALL!" x64 - ) -) else ( - echo [INFO] cl.exe already in PATH, skipping MSVC environment setup. -) - -REM ----------------------------------------------------------------------- -REM Step 4: Ensure buf CLI is installed -REM The CI workflow uses bufbuild/buf-action@v1 (also pinned to -REM BUF_VERSION below) to do the same thing. -REM buf is a single self-contained .exe; install it under -REM %LOCALAPPDATA%\buf\bin\buf.exe to avoid requiring admin rights. -REM ----------------------------------------------------------------------- -set "BUF_VERSION=1.67.0" -set "BUF_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where buf.exe >nul 2>&1 - if not errorlevel 1 set "BUF_FOUND=1" -) -if "%BUF_FOUND%"=="0" ( - echo [INFO] buf.exe not found. Installing buf %BUF_VERSION%... - set "BUF_DIR=%LOCALAPPDATA%\buf\bin" - set "BUF_EXE=!BUF_DIR!\buf.exe" - set "BUF_URL=https://github.com/bufbuild/buf/releases/download/v%BUF_VERSION%/buf-Windows-x86_64.exe" - if "%DRY_RUN%"=="0" ( - if not exist "!BUF_DIR!" mkdir "!BUF_DIR!" - powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('!BUF_URL!','!BUF_EXE!')" - if not exist "!BUF_EXE!" ( - echo [ERROR] Failed to download buf from !BUF_URL!. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: download !BUF_URL! to !BUF_EXE! - ) - REM Add buf to current session PATH - set "PATH=!BUF_DIR!;%PATH%" - REM Persist buf path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"buf\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!BUF_DIR!;!USR_PATH!" - echo [INFO] buf path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!BUF_DIR!;..." - ) - echo [INFO] buf installed successfully. -) else ( - echo [INFO] buf.exe already in PATH. -) - -echo [INFO] Build environment ready. - -REM Export PATH and key MSVC vars back to the caller's environment -endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" diff --git a/test/buf.lock b/test/buf.lock index 13c53ed2..5b5e4348 100644 --- a/test/buf.lock +++ b/test/buf.lock @@ -2,5 +2,5 @@ version: v2 deps: - name: buf.build/tableauio/tableau - commit: 04b3a2c59b6644318d341d4f2220a578 - digest: b5:59bca610664e985502469393a4d47370da9d40d874312a76db4fe133fe61e27fc23b72d2ad45d67c3af76e6620ab1c86c3de0bd13d3517abf6e8b75a346b78fd + commit: 276d7564e86d4bcc80c493e28c5a3897 + digest: b5:507b516066e0e658f97a12e77021a399400baf6acd501c720e7650f711963eae8db5fedf2ae36bce32cfdcd42a81de9fe5c9ebbca7f57fcdae781a562b1a3d65 diff --git a/test/buf.yaml b/test/buf.yaml index d6995604..a7a04953 100644 --- a/test/buf.yaml +++ b/test/buf.yaml @@ -1,5 +1,5 @@ version: v2 deps: - - buf.build/tableauio/tableau:v0.15.1 + - buf.build/tableauio/tableau:v0.16.0 modules: - path: proto diff --git a/test/cpp-tableau-loader/CMakeLists.txt b/test/cpp-tableau-loader/CMakeLists.txt index af39f0ac..7eaa5666 100644 --- a/test/cpp-tableau-loader/CMakeLists.txt +++ b/test/cpp-tableau-loader/CMakeLists.txt @@ -32,38 +32,43 @@ else() endif() # google protobuf -# Try to find protobuf from local submodule first, then fallback to system. -# Protobuf's install.cmake generates config files under: -# - Linux: build/lib64/cmake/protobuf/ (via CMAKE_INSTALL_LIBDIR) -# - Windows: build/cmake/ (MSVC uses "cmake" directly) -# Setting CMAKE_PREFIX_PATH to the build dir allows CMake to search both layouts. -set(LOCAL_PROTOBUF_INSTALL_DIR "${PROJECT_SOURCE_DIR}/../../third_party/_submodules/protobuf/.build/_install") -set(LOCAL_PROTOBUF_SRC_DIR "${PROJECT_SOURCE_DIR}/../../third_party/_submodules/protobuf/src") -if(EXISTS "${LOCAL_PROTOBUF_INSTALL_DIR}" AND EXISTS "${LOCAL_PROTOBUF_SRC_DIR}") - message(STATUS "Found local protobuf submodule, using it preferentially.") - list(PREPEND CMAKE_PREFIX_PATH "${LOCAL_PROTOBUF_INSTALL_DIR}") -endif() - -# Use CONFIG mode explicitly to pick up protobuf's own protobuf-config.cmake -# instead of CMake's built-in FindProtobuf.cmake module, which may not handle -# the local build directory layout correctly. +# +# This project does NOT vendor or build protobuf. Bring your own toolchain +# (matching protoc + libprotobuf, since protobuf v22+ enforces a strict +# gencode/runtime version check via PROTOBUF_VERSION in the generated headers). +# +# Common ways to point CMake at your protobuf install: +# - vcpkg (recommended, cross-platform): +# cmake -S . -B build \ +# -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake +# - System package (apt's libprotobuf-dev, Homebrew's protobuf, ...): no +# extra flag needed; find_package picks it up from the default prefixes. +# - Custom install prefix: +# cmake -S . -B build -DCMAKE_PREFIX_PATH=/path/to/protobuf-install +# (or -DProtobuf_ROOT=/path/to/protobuf-install) +# +# CONFIG mode is used explicitly so we get protobuf's own protobuf-config.cmake +# (and the modern protobuf::libprotobuf imported target with proper +# INTERFACE_INCLUDE_DIRECTORIES) instead of CMake's bundled FindProtobuf.cmake. find_package(Protobuf CONFIG REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") # GoogleTest via FetchContent, pinned to a stable release. Using FetchContent -# (instead of add_subdirectory on protobuf's bundled googletest) gives a -# consistent test framework regardless of which protobuf version is in use. +# (instead of relying on a system gtest) gives a consistent test framework +# regardless of which protobuf version is in use. # # gtest_force_shared_crt: # OFF -> let CMAKE_MSVC_RUNTIME_LIBRARY decide (we set it to MultiThreaded[Debug] # above, i.e. static CRT /MT or /MTd). # ON -> force googletest to use the DYNAMIC CRT (/MD or /MDd). -# Our protobuf submodule is built with protobuf_MSVC_STATIC_RUNTIME=ON (static -# CRT), and our loader_lib also targets static CRT. Forcing gtest to a different -# CRT would mix two C runtimes inside loader.exe; the link may still succeed but -# global-destructor sequencing at process exit can hit an Access Violation -# (gtest's STL objects holding handles allocated by a different heap). Keep gtest -# on the same static CRT as everything else. +# On Windows we expect protobuf to come from vcpkg's x64-windows-static triplet +# (or any other build that uses the static CRT) so that loader_lib, gtest and +# libprotobuf all share one CRT. Mixing CRTs inside loader.exe might still link +# but global-destructor sequencing at process exit can hit an Access Violation +# (gtest's STL objects holding handles allocated by a different heap). If you +# bring a libprotobuf built against the dynamic CRT, also pass +# -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$:Debug>DLL and +# flip gtest_force_shared_crt to ON to keep everything in sync. include(FetchContent) set(gtest_force_shared_crt OFF CACHE BOOL "" FORCE) FetchContent_Declare( diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index f7018a18..e39634a4 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -69,7 +69,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::pa const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + const tableau::WorksheetOptions& worksheet_options = util::GetExtension(descriptor->options(), tableau::worksheet); if (worksheet_options.patch() != tableau::PATCH_NONE) { return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index 87cdbfb0..7146119b 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -69,6 +69,21 @@ const std::string& Format2Ext(Format fmt) { #undef GetMessage #endif +// MapKeyFd returns the key FieldDescriptor of a map-entry message. +// +// ``Descriptor::map_key()`` is a convenience wrapper added in protobuf +// v3.12.0; on older runtimes we fall back to ``field(0)``, which is +// equivalent because map-entry messages always synthesize the key as +// field 0 (and value as field 1) per the proto3 wire-format contract. +inline const google::protobuf::FieldDescriptor* MapKeyFd( + const google::protobuf::Descriptor* map_entry) { +#if GOOGLE_PROTOBUF_VERSION < 3012000 + return map_entry->field(0); +#else + return map_entry->map_key(); +#endif +} + // PatchMessage patches src into dst, which must be a message with the same descriptor. // // # Default PatchMessage mechanism @@ -111,7 +126,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag // Iterates over every populated field. for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + const tableau::FieldOptions& opts = GetExtension(fd->options(), tableau::field); tableau::Patch patch = opts.prop().patch(); if (patch == tableau::PATCH_REPLACE) { dst_reflection->ClearField(&dst, fd); @@ -119,7 +134,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag if (fd->is_map()) { // Reference: // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); + auto key_fd = MapKeyFd(fd->message_type()); int src_count = src_reflection->FieldSize(src, fd); int dst_count = dst_reflection->FieldSize(dst, fd); switch (key_fd->cpp_type()) { diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index 39376272..8ef011cb 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -11,6 +11,7 @@ #include #include #include +#include // Protobuf versions before v4.22.0 (GOOGLE_PROTOBUF_VERSION < 4022000) use the legacy // logging interface (LogLevel, SetLogHandler). Newer versions removed it in favor of Abseil logging. @@ -74,6 +75,29 @@ const std::string& Format2Ext(Format fmt); // PatchMessage patches src into dst, which must be a message with the same descriptor. bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); +// GetExtension reads a typed custom-option extension off any *Options +// message (e.g. MessageOptions, FieldOptions). +// +// On protobuf runtimes < v3.15.0 a static-initialization-order quirk can +// leave the extension payload parked in the options' unknown_fields, so +// `options.GetExtension(id)` returns the default instance. We work around +// it by serializing and reparsing into a fresh OptionsT at call time, by +// which point all extension registrations have run. +// Reference: https://github.com/protocolbuffers/protobuf/releases/tag/v3.15.0 +template +inline auto GetExtension(const OptionsT& options, const ExtT& id) + -> typename std::decay::type { +#if GOOGLE_PROTOBUF_VERSION < 3015000 + OptionsT reparsed; + std::string buf; + options.SerializeToString(&buf); + reparsed.ParseFromString(buf); + return reparsed.GetExtension(id); +#else + return options.GetExtension(id); +#endif +} + #if TABLEAU_PB_LOG_LEGACY // ProtobufLogHandler redirects protobuf internal logs to tableau logger. // Only available for protobuf < v4.22.0, as newer versions removed the old logging interface. diff --git a/test_make.py b/test_make.py new file mode 100644 index 00000000..a1acfccb --- /dev/null +++ b/test_make.py @@ -0,0 +1,1222 @@ +"""Unit + dry-run integration tests for make.py. + +Two layers: + +1. **Unit tests** — import make.py directly and exercise pure logic + (Versions, Platform, _winquote, Runner, etc.). No subprocess, no + network, no filesystem mutation outside pytest's tmp_path. + +2. **Dry-run snapshot tests** — spawn `python3 make.py --dry-run ` + and assert the printed command sequence. Canonical contract test + for "the orchestrator still emits the right cmake/ctest/buf calls." + +Subprocess tests use `--dry-run`, so they never touch the toolchain or +network. The whole suite runs in ~5s on any host. + +Run: + pip install pytest + python -m pytest test_make.py -v +""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +import make + +REPO_ROOT = Path(__file__).resolve().parent +MAKE_PY = REPO_ROOT / "make.py" + + +# --------------------------------------------------------------------------- +# Unit: Versions +# --------------------------------------------------------------------------- + + +class TestVersions: + def test_loads_real_versions_env(self): + v = make.Versions.load(REPO_ROOT) + # Every key documented in versions.env must be present. + for key in ( + "GO_VERSION", + "BUF_VERSION", + "DEFAULT_VARIANT", + "MODERN_PROTOBUF_VERSION", + "MODERN_VCPKG_BASELINE_COMMIT", + "LEGACY_V3_PROTOBUF_VERSION", + "LEGACY_V3_VCPKG_BASELINE_COMMIT", + "DOTNET_VERSION", + "CMAKE_VERSION", + ): + assert key in v.raw, f"Expected {key} in versions.env" + assert v.raw[key], f"{key} must not be empty" + + def test_typed_accessors(self): + v = make.Versions.load(REPO_ROOT) + assert v.go_version == v.raw["GO_VERSION"] + assert v.buf_version == v.raw["BUF_VERSION"] + assert v.dotnet_version == v.raw["DOTNET_VERSION"] + assert v.cmake_version == v.raw["CMAKE_VERSION"] + # default_variant is lowercased. + assert v.default_variant == v.raw["DEFAULT_VARIANT"].lower() + # protobuf_version / vcpkg_baseline_commit resolve through DEFAULT_VARIANT + # to the corresponding MODERN_/LEGACY_V3_-prefixed key. + prefix = v.default_variant.upper().replace("-", "_") + assert v.protobuf_version == v.raw[f"{prefix}_PROTOBUF_VERSION"] + assert v.vcpkg_baseline_commit == v.raw[f"{prefix}_VCPKG_BASELINE_COMMIT"] + + def test_variants_enumerates_all_rows(self): + v = make.Versions.load(REPO_ROOT) + variants = v.variants() + # Both rows declared in versions.env must be present and complete. + assert "modern" in variants + assert "legacy_v3" in variants + for label, row in variants.items(): + assert row.get("protobuf_version"), f"{label} missing protobuf_version" + assert row.get("vcpkg_baseline_commit"), f"{label} missing vcpkg_baseline_commit" + + def test_default_variant_falls_back_to_modern(self): + # Old versions.env files without DEFAULT_VARIANT default to 'modern'. + v = make.Versions(raw={}) + assert v.default_variant == "modern" + + def test_variant_value_falls_back_to_unprefixed_key(self): + # Backward compat: a flat versions.env (legacy schema) still resolves. + v = make.Versions(raw={"PROTOBUF_VERSION": "9.9.9", "VCPKG_BASELINE_COMMIT": "f" * 40}) + assert v.protobuf_version == "9.9.9" + assert v.vcpkg_baseline_commit == "f" * 40 + + def test_default_variant_label_with_dash_resolves(self): + # 'legacy-v3' (CI label form) and 'legacy_v3' (env-key form) both work. + v = make.Versions(raw={ + "DEFAULT_VARIANT": "legacy-v3", + "LEGACY_V3_PROTOBUF_VERSION": "3.21.12", + "LEGACY_V3_VCPKG_BASELINE_COMMIT": "a" * 40, + }) + assert v.protobuf_version == "3.21.12" + assert v.vcpkg_baseline_commit == "a" * 40 + + def test_protobuf_version_looks_like_semver(self): + v = make.Versions.load(REPO_ROOT) + # Cheap smoke check — guards against a typo that breaks vcpkg. + parts = v.protobuf_version.split(".") + assert 2 <= len(parts) <= 3 + for p in parts: + assert p.isdigit(), f"Non-numeric protobuf version segment: {p}" + + def test_vcpkg_baseline_is_full_sha(self): + v = make.Versions.load(REPO_ROOT) + assert len(v.vcpkg_baseline_commit) == 40 + int(v.vcpkg_baseline_commit, 16) # raises ValueError if not hex + + def test_ignores_blank_and_comment_lines(self, tmp_path): + env = tmp_path / ".devcontainer" + env.mkdir() + (env / "versions.env").write_text( + "\n" + "# this is a comment\n" + "FOO=bar\n" + "\n" + "# another comment\n" + "BAZ=qux\n", + encoding="utf-8", + ) + v = make.Versions.load(tmp_path) + assert v.raw == {"FOO": "bar", "BAZ": "qux"} + + def test_missing_file_raises(self, tmp_path): + with pytest.raises(FileNotFoundError): + make.Versions.load(tmp_path) + + def test_get_with_default(self): + v = make.Versions(raw={"FOO": "bar"}) + assert v.get("FOO") == "bar" + assert v.get("MISSING") is None + assert v.get("MISSING", "default") == "default" + + +# --------------------------------------------------------------------------- +# Unit: Platform +# --------------------------------------------------------------------------- + + +class TestPlatform: + def test_detect_returns_one_os(self): + p = make.Platform.detect() + # Exactly one of is_windows / is_macos / is_linux must be True. + flags = [p.is_windows, p.is_macos, p.is_linux] + assert sum(flags) == 1, f"Expected exactly one OS flag, got {flags}" + + def test_vcpkg_triplet_windows(self): + p = make.Platform(sys_platform="win32", machine="amd64", in_devcontainer=False) + assert p.vcpkg_triplet == "x64-windows-static" + + def test_vcpkg_triplet_macos_x64(self): + p = make.Platform( + sys_platform="darwin", machine="x86_64", in_devcontainer=False + ) + assert p.vcpkg_triplet == "x64-osx" + + def test_vcpkg_triplet_macos_arm64(self): + p = make.Platform(sys_platform="darwin", machine="arm64", in_devcontainer=False) + assert p.vcpkg_triplet == "arm64-osx" + + def test_vcpkg_triplet_linux_x64(self): + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=False) + assert p.vcpkg_triplet == "x64-linux" + + def test_vcpkg_triplet_linux_arm64(self): + p = make.Platform( + sys_platform="linux", machine="aarch64", in_devcontainer=False + ) + assert p.vcpkg_triplet == "arm64-linux" + + def test_cmake_toolchain_args_devcontainer_is_empty(self): + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=True) + assert p.cmake_toolchain_args() == [] + + def test_cmake_toolchain_args_linux_no_vcpkg_is_empty(self, monkeypatch): + # Without vcpkg installed (no VCPKG_ROOT), cmake falls back to + # system protobuf via find_package — no -D flags needed. + monkeypatch.delenv("VCPKG_ROOT", raising=False) + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=False) + assert p.cmake_toolchain_args() == [] + + def test_cmake_toolchain_args_linux_with_vcpkg_emits_flags(self, tmp_path): + # After `make.py setup --lang cpp` populates ~/vcpkg, native Linux + # builds also route through vcpkg. Same protobuf pin as Windows. + p = make.Platform( + sys_platform="linux", + machine="x86_64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args() + assert any("CMAKE_TOOLCHAIN_FILE" in a for a in args) + assert any("VCPKG_TARGET_TRIPLET=x64-linux" in a for a in args) + + def test_cmake_toolchain_args_macos_no_vcpkg_is_empty(self, monkeypatch): + monkeypatch.delenv("VCPKG_ROOT", raising=False) + p = make.Platform(sys_platform="darwin", machine="arm64", in_devcontainer=False) + assert p.cmake_toolchain_args() == [] + + def test_cmake_toolchain_args_macos_with_vcpkg_emits_flags(self, tmp_path): + p = make.Platform( + sys_platform="darwin", + machine="arm64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args() + assert any("CMAKE_TOOLCHAIN_FILE" in a for a in args) + assert any("VCPKG_TARGET_TRIPLET=arm64-osx" in a for a in args) + + def test_cmake_toolchain_args_windows_with_vcpkg_root(self, tmp_path): + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args() + assert any("CMAKE_TOOLCHAIN_FILE" in a for a in args) + assert any("VCPKG_TARGET_TRIPLET=x64-windows-static" in a for a in args) + # Path must include vcpkg.cmake. + toolchain_arg = next(a for a in args if "CMAKE_TOOLCHAIN_FILE" in a) + assert toolchain_arg.endswith("vcpkg.cmake") + + def test_cmake_toolchain_args_windows_explicit_triplet(self, tmp_path): + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args(triplet="x64-windows") + assert any("VCPKG_TARGET_TRIPLET=x64-windows" in a for a in args) + assert not any("VCPKG_TARGET_TRIPLET=x64-windows-static" in a for a in args) + + def test_cmake_toolchain_args_windows_no_vcpkg_root_returns_empty( + self, monkeypatch + ): + # No VCPKG_ROOT in env, no vcpkg_root attr -> empty list (lets cmake + # fail with a useful "Could not find Protobuf" error). + monkeypatch.delenv("VCPKG_ROOT", raising=False) + p = make.Platform(sys_platform="win32", machine="amd64", in_devcontainer=False) + assert p.cmake_toolchain_args() == [] + + + def test_detect_ignores_dockerenv_marker(self, monkeypatch): + """Bug fix: /.dockerenv is created by Docker for *every* container, + not just the loader devcontainer. The detect() heuristic must NOT + treat its presence as a devcontainer signal.""" + # Simulate: only /.dockerenv exists; /opt/vcpkg/active does NOT. + # Use as_posix() so the literal compares correctly on Windows where + # str(WindowsPath('/.dockerenv')) renders with backslashes. + def fake_exists(self): + return self.as_posix() == "/.dockerenv" + monkeypatch.setattr(make.Path, "exists", fake_exists) + p = make.Platform.detect() + assert p.in_devcontainer is False, ( + "/.dockerenv alone must NOT be treated as devcontainer" + ) + + def test_detect_recognizes_vcpkg_active_marker(self, monkeypatch): + """Positive: /opt/vcpkg/active is the marker the devcontainer's + Dockerfile actually sets. Its presence is the sole devcontainer signal.""" + def fake_exists(self): + return self.as_posix() == "/opt/vcpkg/active" + monkeypatch.setattr(make.Path, "exists", fake_exists) + p = make.Platform.detect() + assert p.in_devcontainer is True + + +# --------------------------------------------------------------------------- +# Unit: Platform.windows_msvc_wrap +# --------------------------------------------------------------------------- + + +class TestWindowsMsvcWrap: + """Exercises the central insight of make.py: the cmd-shell-string + sentinel that bypasses Python's CreateProcess quoting on Windows.""" + + def test_passthrough_on_linux(self): + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=False) + cmd = ["cmake", "-S", ".", "-B", "build"] + assert p.windows_msvc_wrap(cmd) == cmd + + def test_passthrough_on_macos(self): + p = make.Platform(sys_platform="darwin", machine="arm64", in_devcontainer=False) + cmd = ["cmake", "-S", ".", "-B", "build"] + assert p.windows_msvc_wrap(cmd) == cmd + + def test_windows_with_no_vcvarsall_returns_unchanged(self): + # When MSVC isn't installed, return cmd unchanged so subprocess + # fails with a useful "cl.exe not found" error rather than a + # confusing wrapping failure. + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcvarsall_path=None, + ) + # Stub locate_vcvarsall to return None. + original = make.locate_vcvarsall + try: + make.locate_vcvarsall = lambda: None + cmd = ["cmake", "-S", "."] + assert p.windows_msvc_wrap(cmd) == cmd + finally: + make.locate_vcvarsall = original + + def test_windows_with_vcvarsall_emits_sentinel(self, tmp_path): + fake_vcvars = tmp_path / "vcvarsall.bat" + fake_vcvars.write_text("rem fake", encoding="utf-8") + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcvarsall_path=fake_vcvars, + ) + wrapped = p.windows_msvc_wrap(["buf", "generate", ".."]) + assert len(wrapped) == 1 + line = wrapped[0] + assert line.startswith(make._WIN_SHELL_MARKER) + # The shell line must `call` vcvarsall and then `&&` the inner cmd. + body = line[len(make._WIN_SHELL_MARKER) :] + assert body.startswith(f'call "{fake_vcvars}" x64') + assert " >nul && buf generate .." in body + + def test_windows_quotes_paths_with_spaces(self, tmp_path): + fake_vcvars = tmp_path / "vcvarsall.bat" + fake_vcvars.write_text("rem fake", encoding="utf-8") + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcvarsall_path=fake_vcvars, + ) + wrapped = p.windows_msvc_wrap(["cmake", "-DPATH=C:\\Program Files\\foo"]) + body = wrapped[0][len(make._WIN_SHELL_MARKER) :] + # The arg with a space MUST be quoted, but the bare cmake should not. + assert ( + ' cmake "-DPATH=C:\\Program Files\\foo"' in body + or ' cmake -DPATH="C:\\Program Files\\foo"' in body + or ' cmake "-DPATH=C:\\Program Files\\foo"' in body + or '"-DPATH=C:\\Program Files\\foo"' in body + ) # at least one of these forms + + +# --------------------------------------------------------------------------- +# Unit: _winquote +# --------------------------------------------------------------------------- + + +class TestQuoting: + def test_empty_string(self): + assert make._winquote("") == '""' + + def test_simple_arg_unchanged(self): + assert make._winquote("foo") == "foo" + assert make._winquote("--flag=value") == "--flag=value" + + def test_arg_with_space_is_quoted(self): + assert make._winquote("hello world") == '"hello world"' + + def test_arg_with_paren_is_quoted(self): + # The (x86) parser footgun from xxx.bat — must be quoted. + assert make._winquote("C:\\Program Files (x86)\\foo").startswith('"') + + def test_arg_with_embedded_quote_is_doubled(self): + # cmd quotes embedded " by doubling. + assert make._winquote('say "hi"') == '"say ""hi"""' + + +# --------------------------------------------------------------------------- +# Unit: Runner +# --------------------------------------------------------------------------- + + +class TestRunner: + def test_dry_run_does_not_execute(self, capsys, tmp_path): + runner = make.Runner(verbose=False, dry_run=True) + # If this were actually executed, it would create a file we can detect. + marker = tmp_path / "should_not_exist.txt" + if sys.platform == "win32": + cmd = ["cmd", "/c", f"echo hi > {marker}"] + else: + cmd = ["sh", "-c", f"echo hi > {marker}"] + rc = runner.run(cmd) + assert rc == 0 + assert not marker.exists(), "dry-run should not have executed" + out = capsys.readouterr().out + assert "[dry-run]" in out + + def test_dry_run_prints_cwd(self, capsys, tmp_path): + runner = make.Runner(verbose=False, dry_run=True) + runner.run(["echo", "hi"], cwd=tmp_path) + out = capsys.readouterr().out + assert f"cwd={tmp_path}" in out + + def test_sentinel_routes_through_shell(self, capsys): + runner = make.Runner(verbose=False, dry_run=True) + sentinel_cmd = [make._WIN_SHELL_MARKER + "echo hello-world"] + rc = runner.run(sentinel_cmd) + assert rc == 0 + out = capsys.readouterr().out + # The marker must be stripped from the printed line. + assert make._WIN_SHELL_MARKER not in out + assert "echo hello-world" in out + + def test_rmtree_dry_run_does_not_remove(self, capsys, tmp_path): + target = tmp_path / "doomed" + target.mkdir() + (target / "file.txt").write_text("x") + runner = make.Runner(verbose=False, dry_run=True) + runner.rmtree(target) + assert target.exists() + out = capsys.readouterr().out + assert "[dry-run] rm -rf" in out + + def test_rmtree_real_removes(self, tmp_path): + target = tmp_path / "doomed" + target.mkdir() + (target / "file.txt").write_text("x") + runner = make.Runner(verbose=False, dry_run=False) + runner.rmtree(target) + assert not target.exists() + + def test_rmtree_no_op_on_missing(self, tmp_path): + runner = make.Runner(verbose=False, dry_run=False) + runner.rmtree(tmp_path / "does-not-exist") # should not raise + + def test_check_raises_on_failure(self): + runner = make.Runner(verbose=False, dry_run=False) + # `python -c "raise SystemExit(7)"` will exit 7. + with pytest.raises(SystemExit): + runner.run([sys.executable, "-c", "raise SystemExit(7)"]) + + def test_check_false_suppresses_failure(self): + runner = make.Runner(verbose=False, dry_run=False) + rc = runner.run([sys.executable, "-c", "raise SystemExit(7)"], check=False) + assert rc == 7 + + +# --------------------------------------------------------------------------- +# Unit: vcpkg protobuf pin (baseline vs. overrides) +# --------------------------------------------------------------------------- + + +class TestVcpkgProtobufPin: + """_resolve_vcpkg_protobuf_pin: baseline vs. overrides selection. + + The git-history search (_resolve_vcpkg_baseline_for_protobuf) is stubbed + so these stay hermetic — we only assert the floor-based routing. + """ + + def _patch_resolver(self, monkeypatch): + calls = [] + + def fake_resolve(version, vcpkg_root, runner): + calls.append(version) + return f"sha-for-{version}" + + monkeypatch.setattr( + make, "_resolve_vcpkg_baseline_for_protobuf", fake_resolve + ) + return calls + + def test_at_or_above_floor_pins_exact_baseline(self, monkeypatch): + calls = self._patch_resolver(monkeypatch) + runner = make.Runner(verbose=False, dry_run=True) + pin = make._resolve_vcpkg_protobuf_pin("3.21.12", Path("/vcpkg"), runner) + assert pin.baseline == "sha-for-3.21.12" + assert pin.override_version is None # no overrides needed + assert calls == ["3.21.12"] + + def test_floor_itself_has_no_override(self, monkeypatch): + self._patch_resolver(monkeypatch) + runner = make.Runner(verbose=False, dry_run=True) + pin = make._resolve_vcpkg_protobuf_pin("3.14.0", Path("/vcpkg"), runner) + assert pin.baseline == "sha-for-3.14.0" + assert pin.override_version is None + + def test_below_floor_pins_floor_baseline_plus_override(self, monkeypatch): + calls = self._patch_resolver(monkeypatch) + runner = make.Runner(verbose=False, dry_run=True) + pin = make._resolve_vcpkg_protobuf_pin("3.12.0", Path("/vcpkg"), runner) + # builtin-baseline resolves the FLOOR, not the requested version. + assert pin.baseline == "sha-for-3.14.0" + assert pin.override_version == "3.12.0" + assert calls == ["3.14.0"] + + def test_floor_str_matches_tuple(self): + assert make._VCPKG_PROTOBUF_BASELINE_FLOOR_STR == "3.14.0" + assert ( + make._parse_protobuf_version_tuple( + make._VCPKG_PROTOBUF_BASELINE_FLOOR_STR + ) + == make._VCPKG_PROTOBUF_BASELINE_FLOOR + ) + + # ----- below-floor existence validation (fail fast) ----- + + def test_below_floor_missing_raises_before_baseline_resolve(self, monkeypatch): + # A below-floor version absent from the registry must raise *before* + # any baseline git-search happens (so callers fail fast, trees intact). + baseline_calls = self._patch_resolver(monkeypatch) + monkeypatch.setattr( + make, "_protobuf_version_in_registry", lambda root, v: False + ) + runner = make.Runner(verbose=False, dry_run=False) + with pytest.raises(RuntimeError) as exc: + make._resolve_vcpkg_protobuf_pin("3.12.99", Path("/vcpkg"), runner) + # Never reached the baseline resolver. + assert baseline_calls == [] + assert "3.12.99" in str(exc.value) + assert "not present" in str(exc.value) + + def test_below_floor_present_resolves(self, monkeypatch): + self._patch_resolver(monkeypatch) + monkeypatch.setattr( + make, "_protobuf_version_in_registry", lambda root, v: True + ) + runner = make.Runner(verbose=False, dry_run=False) + pin = make._resolve_vcpkg_protobuf_pin("3.12.0", Path("/vcpkg"), runner) + assert pin.baseline == "sha-for-3.14.0" + assert pin.override_version == "3.12.0" + + def test_below_floor_dry_run_skips_existence_check(self, monkeypatch): + # In dry-run there is no real checkout, so the registry must NOT be + # consulted — the pin should resolve via the placeholder path. + self._patch_resolver(monkeypatch) + + def boom(root, v): # pragma: no cover - must never be called + raise AssertionError("registry consulted in dry-run") + + monkeypatch.setattr(make, "_protobuf_version_in_registry", boom) + runner = make.Runner(verbose=False, dry_run=True) + pin = make._resolve_vcpkg_protobuf_pin("3.12.99", Path("/vcpkg"), runner) + assert pin.override_version == "3.12.99" + + def test_resolve_below_floor_override_returns_none_above_floor(self, monkeypatch): + runner = make.Runner(verbose=False, dry_run=False) + assert ( + make._resolve_below_floor_override("3.21.12", Path("/vcpkg"), runner) + is None + ) + + +# --------------------------------------------------------------------------- +# Unit: protobuf hard support floor (3.8.0) + --force +# --------------------------------------------------------------------------- + + +class TestProtobufMinSupported: + """_check_protobuf_min_supported: refuse < 3.8.0 unless forced.""" + + def test_below_floor_raises_without_force(self): + with pytest.raises(RuntimeError) as exc: + make._check_protobuf_min_supported("3.5.0", force=False) + msg = str(exc.value) + assert "3.5.0" in msg + assert make._PROTOBUF_MIN_SUPPORTED_STR in msg # "3.8.0" + assert "--force" in msg + + def test_below_floor_with_force_warns_and_returns(self, capsys): + # Forced: must NOT raise; emits a warning to stderr instead. + make._check_protobuf_min_supported("3.5.0", force=True) + err = capsys.readouterr().err + assert "3.5.0" in err + assert "force" in err.lower() + + def test_at_floor_is_allowed(self): + # 3.8.0 itself is supported (boundary inclusive) — no raise, no warning. + make._check_protobuf_min_supported("3.8.0", force=False) + + def test_above_floor_is_allowed(self): + make._check_protobuf_min_supported("3.21.12", force=False) + + def test_unparseable_version_is_allowed(self): + # Defensive: a non-numeric version is left for downstream resolution + # rather than rejected by the numeric floor guard. + make._check_protobuf_min_supported("main", force=False) + + def test_floor_str_matches_tuple(self): + assert make._PROTOBUF_MIN_SUPPORTED_STR == "3.8.0" + assert ( + make._parse_protobuf_version_tuple(make._PROTOBUF_MIN_SUPPORTED_STR) + == make._PROTOBUF_MIN_SUPPORTED + ) + + def test_support_floor_not_above_baseline_floor(self): + # The support floor must sit at/below the baseline floor so the + # overrides range it gates [support, baseline) is non-empty. + assert make._PROTOBUF_MIN_SUPPORTED <= make._VCPKG_PROTOBUF_BASELINE_FLOOR + + +class TestKnownVersionsHintFloor: + """_format_known_protobuf_versions_hint: below-floor listing reaches 3.8.0. + + The vcpkg version registry read is stubbed so the test is hermetic and + asserts only the floor-filtering / no-truncation behavior. + """ + + # Newest-first registry, spanning above and below both floors plus a couple + # of pre-3.8.0 entries that must be dropped. + _REGISTRY = [ + "6.33.4", "5.29.5", "4.25.1", "3.21.12", "3.18.0", "3.15.8", + "3.14.0", "3.13.0", "3.11.0", "3.9.0", "3.8.0", "3.7.0", "3.5.0", + ] + + def _patch_registry(self, monkeypatch): + monkeypatch.setattr( + make, + "_read_protobuf_registry_versions", + lambda root: list(self._REGISTRY), + ) + + def _listing_line(self, hint): + # The single line that enumerates the versions (starts with the + # "known protobuf versions" bullet), isolated from header notes that + # also mention the floor strings. + for line in hint.splitlines(): + if "known protobuf versions" in line: + return line.split("): ", 1)[1] + raise AssertionError("no version-listing line in hint") + + def test_include_below_floor_lists_down_to_support_floor(self, monkeypatch): + self._patch_registry(monkeypatch) + hint = make._format_known_protobuf_versions_hint( + Path("/vcpkg"), include_below_floor=True + ) + listed = self._listing_line(hint) + # Reaches the support floor... + assert "3.8.0" in listed + # ...but excludes anything below it. + assert "3.7.0" not in listed + assert "3.5.0" not in listed + # No truncation marker — the full in-range list is shown. + assert "older)" not in hint + # Below-baseline versions (>= 3.8, < 3.14) are present. + assert "3.9.0" in listed and "3.11.0" in listed + + def test_baseline_only_listing_still_filters_to_baseline_floor( + self, monkeypatch + ): + self._patch_registry(monkeypatch) + hint = make._format_known_protobuf_versions_hint( + Path("/vcpkg"), include_below_floor=False + ) + listed = self._listing_line(hint) + # Default listing only surfaces >= 3.14.0 (usable as builtin-baseline). + assert "3.14.0" in listed + assert "3.13.0" not in listed + assert "3.8.0" not in listed + + +# --------------------------------------------------------------------------- +# Unit: repo root discovery +# --------------------------------------------------------------------------- + + +class TestRepoRoot: + def test_finds_root_from_repo(self): + assert make.find_repo_root() == REPO_ROOT + + def test_finds_root_from_subdir(self): + # Walk into a deep subdir and ensure we still locate it. + deep = REPO_ROOT / "internal" / "options" + if deep.is_dir(): + assert make.find_repo_root(deep) == REPO_ROOT + + def test_raises_outside_repo(self, tmp_path): + with pytest.raises(SystemExit): + make.find_repo_root(tmp_path) + + +# --------------------------------------------------------------------------- +# Unit: lang directory mapping +# --------------------------------------------------------------------------- + + +class TestLangDir: + def test_go(self): + assert ( + make._lang_dir(REPO_ROOT, "go") == REPO_ROOT / "test" / "go-tableau-loader" + ) + + def test_cpp(self): + assert ( + make._lang_dir(REPO_ROOT, "cpp") + == REPO_ROOT / "test" / "cpp-tableau-loader" + ) + + def test_csharp(self): + assert ( + make._lang_dir(REPO_ROOT, "csharp") + == REPO_ROOT / "test" / "csharp-tableau-loader" + ) + + +# --------------------------------------------------------------------------- +# Subprocess helpers +# --------------------------------------------------------------------------- + + +def run_make(*args: str, cwd: Path = REPO_ROOT) -> subprocess.CompletedProcess: + """Spawn `python3 make.py ` and capture stdout+stderr.""" + return subprocess.run( + [sys.executable, str(MAKE_PY), *args], + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + + +# --------------------------------------------------------------------------- +# Integration: top-level invocations +# --------------------------------------------------------------------------- + + +class TestTopLevel: + def test_help_exits_zero(self): + proc = run_make("--help") + assert proc.returncode == 0 + assert "make.py" in proc.stdout + assert "setup" in proc.stdout + assert "test" in proc.stdout + + def test_no_args_prints_help(self): + proc = run_make() + assert proc.returncode == 0 + assert "usage:" in proc.stdout.lower() + + def test_version_prints_versions_env(self): + proc = run_make("--version") + assert proc.returncode == 0 + assert f"make.py {make.MAKE_PY_VERSION}" in proc.stdout + # Must echo every key from versions.env. + for key in ( + "GO_VERSION", + "BUF_VERSION", + "DEFAULT_VARIANT", + "MODERN_PROTOBUF_VERSION", + "MODERN_VCPKG_BASELINE_COMMIT", + "LEGACY_V3_PROTOBUF_VERSION", + "LEGACY_V3_VCPKG_BASELINE_COMMIT", + "DOTNET_VERSION", + "CMAKE_VERSION", + ): + assert key in proc.stdout, f"--version missing {key}" + + def test_env_emits_valid_json(self): + proc = run_make("env") + assert proc.returncode == 0 + info = json.loads(proc.stdout) + # Sanity-check structure. + assert info["make_py_version"] == make.MAKE_PY_VERSION + assert "sys_platform" in info + assert "vcpkg_triplet" in info + assert "tools" in info + assert "versions_env" in info + assert info["versions_env"]["GO_VERSION"] + + +# --------------------------------------------------------------------------- +# Integration: dry-run snapshots +# --------------------------------------------------------------------------- + + +class TestDryRunGo: + def test_test_lang_go(self): + proc = run_make("--dry-run", "test", "--lang", "go") + assert proc.returncode == 0 + out = proc.stdout + assert "buf generate .." in out + assert "go test" in out + assert "-timeout" in out and "30m" in out + # -race default depends on host OS; covered separately in + # test_test_lang_go_race_default_matches_host. + + def test_test_lang_go_no_race(self): + proc = run_make("--dry-run", "test", "--lang", "go", "--no-race") + assert proc.returncode == 0 + # When --no-race is passed, "-race" must NOT appear in the go test line. + for line in proc.stdout.splitlines(): + if "go test" in line: + assert "-race" not in line, f"Expected --no-race to drop -race: {line}" + + def test_test_lang_go_race_default_matches_host(self): + # Default --race depends on host OS: + # - Windows -> default OFF (-race needs cgo + C compiler) + # - Linux/macOS -> default ON + proc = run_make("--dry-run", "test", "--lang", "go") + assert proc.returncode == 0 + go_test_lines = [l for l in proc.stdout.splitlines() if "go test" in l] + assert go_test_lines, "no go test line" + joined = "\n".join(go_test_lines) + if sys.platform == "win32": + assert "-race" not in joined, "Windows default should NOT include -race" + else: + assert "-race" in joined, "Linux/macOS default should include -race" + + def test_test_lang_go_explicit_race_forces_on(self): + # Even on Windows, --race explicitly enables it (user opt-in). + proc = run_make("--dry-run", "test", "--lang", "go", "--race") + assert proc.returncode == 0 + go_test_lines = [l for l in proc.stdout.splitlines() if "go test" in l] + joined = "\n".join(go_test_lines) + assert "-race" in joined, "--race must force -race on regardless of host" + + def test_test_lang_go_smoke_skips_buf_generate(self): + proc = run_make("--dry-run", "test", "--lang", "go", "--smoke") + assert proc.returncode == 0 + # Smoke must NOT call buf generate; only go vet on the plugin packages. + assert "buf generate" not in proc.stdout + assert "go vet" in proc.stdout + # Specific plugin packages required. + for pkg in ( + "./cmd/...", + "./pkg/...", + "./internal/options/...", + "./internal/loadutil/...", + "./internal/xproto/...", + ): + assert pkg in proc.stdout, f"smoke missing {pkg}" + + def test_test_lang_go_filter(self): + proc = run_make( + "--dry-run", "test", "--lang", "go", "-k", "Test_ActivityConf_OrderedMap" + ) + assert proc.returncode == 0 + assert "-run" in proc.stdout + assert "Test_ActivityConf_OrderedMap" in proc.stdout + + def test_test_lang_go_coverage(self): + proc = run_make("--dry-run", "test", "--lang", "go", "--coverage") + assert proc.returncode == 0 + assert "-coverprofile=coverage.txt" in proc.stdout + assert "-covermode=atomic" in proc.stdout + + +class TestDryRunCpp: + def test_test_lang_cpp_default(self): + proc = run_make("--dry-run", "test", "--lang", "cpp") + assert proc.returncode == 0 + out = proc.stdout + assert "buf generate .." in out + assert "cmake -S . -B build" in out + assert "-DCMAKE_BUILD_TYPE=Debug" in out + assert "-DCMAKE_CXX_STANDARD=17" in out + assert "cmake --build build --parallel" in out + assert "ctest --test-dir build --output-on-failure" in out + + def test_test_lang_cpp_no_clean_skips_wipe(self): + proc_clean = run_make("--dry-run", "test", "--lang", "cpp") + proc_no_clean = run_make("--dry-run", "test", "--lang", "cpp", "--no-clean") + # Clean run must mention rm-rf the build/src dirs; --no-clean must not. + assert "rm -rf" in proc_clean.stdout + assert "build" in proc_clean.stdout + assert "rm -rf" not in proc_no_clean.stdout + + def test_test_lang_cpp_cxx_std_20(self): + proc = run_make("--dry-run", "test", "--lang", "cpp", "--cxx-std", "20") + assert proc.returncode == 0 + assert "-DCMAKE_CXX_STANDARD=20" in proc.stdout + assert "-DCMAKE_CXX_STANDARD=17" not in proc.stdout + + def test_test_lang_cpp_cxx_compiler_clang(self): + proc = run_make("--dry-run", "test", "--lang", "cpp", "--cxx-compiler", "clang") + assert proc.returncode == 0 + assert "-DCMAKE_CXX_COMPILER=clang++" in proc.stdout + + def test_test_lang_cpp_protobuf_version_manifest_mode(self): + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-windows-static", + "--no-clean", + ) + assert proc.returncode == 0 + out = proc.stdout + # Manifest-mode flags MUST appear when --protobuf-version is set. + assert "-DVCPKG_INSTALLED_DIR=" in out + assert "-DVCPKG_MANIFEST_INSTALL=OFF" in out + # Must also `vcpkg install` the manifest before cmake configure. + assert "vcpkg" in out and "install" in out + # Triplet must be plumbed through. + assert "--triplet=x64-windows-static" in out + + def test_test_lang_cpp_manifest_no_vcpkg_root_dry_run_ok(self, monkeypatch): + # On a host with no VCPKG_ROOT (e.g. CI runner of testing-make.yml), + # --dry-run must still print the command sequence rather than abort. + # Real (non-dry-run) execution still hard-errors via a separate code + # path; that's covered by the unit test in TestPlatform. + monkeypatch.delenv("VCPKG_ROOT", raising=False) + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-linux", + "--no-clean", + ) + assert proc.returncode == 0, proc.stderr + assert "-DVCPKG_INSTALLED_DIR=" in proc.stdout + assert "-DVCPKG_MANIFEST_INSTALL=OFF" in proc.stdout + + def test_test_lang_cpp_manifest_forces_toolchain_on_linux(self, monkeypatch): + # Manifest mode must emit -DCMAKE_TOOLCHAIN_FILE even on Linux: + # find_package(Protobuf) needs vcpkg's vcpkg.cmake to resolve the + # manifest-installed protobuf. Plain Linux (no --protobuf-version) + # would correctly get [] (system protobuf via apt). + monkeypatch.setenv("VCPKG_ROOT", "/tmp/fake-vcpkg") + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-linux", + "--no-clean", + ) + assert proc.returncode == 0, proc.stderr + # Must have toolchain flags for the manifest install to be picked up. + assert "-DCMAKE_TOOLCHAIN_FILE=" in proc.stdout + assert "-DVCPKG_TARGET_TRIPLET=x64-linux" in proc.stdout + + def test_test_lang_cpp_no_vcpkg_install_skips_install(self): + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-windows-static", + "--no-clean", + "--no-vcpkg-install", + ) + assert proc.returncode == 0 + # No `vcpkg ... install ...` line when --no-vcpkg-install is set. + # cmake configure still runs, with the manifest-mode flags. + assert "-DVCPKG_INSTALLED_DIR=" in proc.stdout + for line in proc.stdout.splitlines(): + # `vcpkg install` must not appear; `cmake` calls are fine. + if "vcpkg.exe install" in line or "vcpkg install" in line.replace( + ".exe", "" + ): + pytest.fail(f"--no-vcpkg-install did not skip vcpkg install: {line}") + + def test_test_lang_cpp_filter(self): + proc = run_make( + "--dry-run", "test", "--lang", "cpp", "-k", "HubTest.Load", "--no-clean" + ) + assert proc.returncode == 0 + # ctest -R + ctest_lines = [l for l in proc.stdout.splitlines() if "ctest" in l] + assert ctest_lines, "no ctest line in output" + joined = "\n".join(ctest_lines) + assert "-R" in joined and "HubTest.Load" in joined + + +class TestDryRunCsharp: + def test_test_lang_csharp_default(self): + proc = run_make("--dry-run", "test", "--lang", "csharp") + assert proc.returncode == 0 + assert "buf generate .." in proc.stdout + assert "dotnet test" in proc.stdout + assert "--nologo" in proc.stdout + + def test_test_lang_csharp_filter(self): + proc = run_make("--dry-run", "test", "--lang", "csharp", "-k", "HubTest.Load") + assert proc.returncode == 0 + # dotnet test --filter "FullyQualifiedName~" + assert "FullyQualifiedName~HubTest.Load" in proc.stdout + + +class TestDryRunGenerateAndBuild: + def test_generate_lang_go(self): + proc = run_make("--dry-run", "generate", "--lang", "go") + assert proc.returncode == 0 + assert "buf generate .." in proc.stdout + + def test_generate_lang_cpp(self): + proc = run_make("--dry-run", "generate", "--lang", "cpp") + assert proc.returncode == 0 + assert "buf generate .." in proc.stdout + + def test_build_lang_go_no_test(self): + proc = run_make("--dry-run", "build", "--lang", "go") + assert proc.returncode == 0 + # build (not test) must call `go build`, not `go test`. + assert "go build" in proc.stdout + # Must NOT run tests. + for line in proc.stdout.splitlines(): + assert "go test" not in line, f"build invoked go test: {line}" + + def test_build_lang_csharp_calls_dotnet_build(self): + proc = run_make("--dry-run", "build", "--lang", "csharp") + assert proc.returncode == 0 + assert "dotnet build" in proc.stdout + for line in proc.stdout.splitlines(): + assert "dotnet test" not in line + + +class TestDryRunClean: + def test_clean_cpp(self): + proc = run_make("--dry-run", "clean", "--lang", "cpp") + assert proc.returncode == 0 + out = proc.stdout + # Must wipe all three cpp dirs. + assert "build" in out + assert "src\\tableau" in out or "src/tableau" in out + assert "src\\protoconf" in out or "src/protoconf" in out + + def test_clean_csharp(self): + proc = run_make("--dry-run", "clean", "--lang", "csharp") + assert proc.returncode == 0 + out = proc.stdout + assert "bin" in out + assert "obj" in out + assert "protoconf" in out + + def test_clean_all(self): + proc = run_make("--dry-run", "clean", "--all") + assert proc.returncode == 0 + out = proc.stdout + # Should mention dirs from at least cpp, csharp, go. + assert "cpp-tableau-loader" in out + assert "csharp-tableau-loader" in out + assert "go-tableau-loader" in out + + +# --------------------------------------------------------------------------- +# Integration: setup is a no-op in devcontainer +# --------------------------------------------------------------------------- + + +class TestCrossPlatformPinning: + """Unit tests for the macOS/Linux pinning helpers (Go tarball, buf + binary, vcpkg). Network calls are guarded by --dry-run.""" + + def test_go_arch_macos_intel(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "x86_64") + assert make._go_arch_macos() == "amd64" + + def test_go_arch_macos_apple_silicon(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "arm64") + assert make._go_arch_macos() == "arm64" + + def test_go_arch_linux_x64(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "x86_64") + assert make._go_arch_linux() == "amd64" + + def test_go_arch_linux_arm64(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "aarch64") + assert make._go_arch_linux() == "arm64" + + def test_setup_macos_dispatches_to_vcpkg(self, monkeypatch, capsys): + # Simulate macOS host with brew available. Dry-run so no real install. + # The handler's _setup_vcpkg call should print the cloning hint. + monkeypatch.setattr(make.Platform, "detect", classmethod( + lambda cls: make.Platform(sys_platform="darwin", machine="x86_64", + in_devcontainer=False))) + monkeypatch.setattr(make, "_which", lambda name: "/usr/local/bin/brew" if name == "brew" else None) + ctx = make.Context( + repo_root=REPO_ROOT, + versions=make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + args = type("Args", (), {"lang": "cpp", "skip_vcpkg": False})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + # Must mention vcpkg (cross-platform install path). + assert "vcpkg" in out.lower() + + def test_setup_linux_dispatches_to_vcpkg(self, monkeypatch, capsys): + monkeypatch.setattr(make.Platform, "detect", classmethod( + lambda cls: make.Platform(sys_platform="linux", machine="x86_64", + in_devcontainer=False))) + # Stub _which to make apt-get appear available, brew/dnf absent. + which_table = {"apt-get": "/usr/bin/apt-get"} + monkeypatch.setattr(make, "_which", lambda name: which_table.get(name)) + # Stub os.geteuid (not present on Windows test host). + monkeypatch.setattr(make.os, "geteuid", lambda: 1000, raising=False) + ctx = make.Context( + repo_root=REPO_ROOT, + versions=make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + args = type("Args", (), {"lang": "cpp", "skip_vcpkg": False})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "vcpkg" in out.lower() + + + + def test_setup_skips_inside_devcontainer(self, monkeypatch, tmp_path): + # Simulate devcontainer detection by patching Platform.detect. + original_detect = make.Platform.detect + + @classmethod + def fake_detect(cls): + return make.Platform( + sys_platform="linux", + machine="x86_64", + in_devcontainer=True, + ) + + monkeypatch.setattr(make.Platform, "detect", fake_detect) + try: + # Build a minimal Context and call cmd_setup directly. + ctx = make.Context( + repo_root=REPO_ROOT, + versions=make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + args = type("Args", (), {"lang": "all"})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + finally: + make.Platform.detect = original_detect + + +# --------------------------------------------------------------------------- +# Unit: Windows setup +# --------------------------------------------------------------------------- + + +class TestSetupWindows: + """Windows-specific setup invariants. Runs on every host because + --dry-run intercepts the subprocess; we only assert against the printed + command sequence.""" + + def _windows_ctx(self, monkeypatch, versions=None): + """Pretend we're a clean Windows host: nothing on PATH, no MSVC.""" + monkeypatch.setattr( + make.Platform, + "detect", + classmethod( + lambda cls: make.Platform( + sys_platform="win32", machine="amd64", in_devcontainer=False + ) + ), + ) + monkeypatch.setattr(make, "_which", lambda name: None) + # locate_vcvarsall already returns None off-Windows; explicit for clarity. + monkeypatch.setattr(make, "locate_vcvarsall", lambda: None) + return make.Context( + repo_root=REPO_ROOT, + versions=versions or make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + + def test_installs_go_for_cpp_lang(self, monkeypatch, capsys): + """Regression: --lang cpp on Windows must still install Go. + buf-generate runs the Go protoc plugins via `go run`, so without + Go on PATH every codegen step fails — for every target language.""" + ctx = self._windows_ctx(monkeypatch) + args = type("Args", (), {"lang": "cpp", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "GoLang.Go" in out, "--lang cpp on Windows must still install Go" + + def test_installs_go_for_csharp_lang(self, monkeypatch, capsys): + """Same as above but for --lang csharp.""" + ctx = self._windows_ctx(monkeypatch) + args = type("Args", (), {"lang": "csharp", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "GoLang.Go" in out, "--lang csharp on Windows must still install Go" + + def test_go_winget_id_uses_versions_env_major_minor(self, monkeypatch, capsys): + """winget id must derive from GO_VERSION (not hardcoded) so bumping + versions.env alone is enough to bump the host pin — matches the + macOS/Linux tarball flow which already reads ctx.versions.go_version.""" + versions = make.Versions(raw={"GO_VERSION": "1.99.7", "DOTNET_VERSION": "8.0"}) + ctx = self._windows_ctx(monkeypatch, versions=versions) + args = type("Args", (), {"lang": "go", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + # GoLang.Go winget id is `.` — patch level is dropped. + assert "GoLang.Go.1.99" in out, f"Expected pinned 1.99 winget id; got: {out}" + # Hardcoded 1.24 from the old code path must NOT leak through. + assert "GoLang.Go.1.24" not in out + + def test_dotnet_winget_id_uses_versions_env_major(self, monkeypatch, capsys): + """winget .NET SDK id format is `Microsoft.DotNet.SDK.` — + derive from DOTNET_VERSION instead of hardcoding `.8`.""" + versions = make.Versions(raw={"GO_VERSION": "1.24.0", "DOTNET_VERSION": "9.0"}) + ctx = self._windows_ctx(monkeypatch, versions=versions) + args = type("Args", (), {"lang": "csharp", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "Microsoft.DotNet.SDK.9" in out + assert "Microsoft.DotNet.SDK.8" not in out diff --git a/third_party/_submodules/protobuf b/third_party/_submodules/protobuf deleted file mode 160000 index cc7b1b53..00000000 --- a/third_party/_submodules/protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cc7b1b53234cd7a8f50d90ac3933b240dcf4cd97