diff --git a/.claude/golang-upgrade-process.md b/.claude/golang-upgrade-process.md new file mode 100644 index 000000000..c044ea43b --- /dev/null +++ b/.claude/golang-upgrade-process.md @@ -0,0 +1,113 @@ +# Go Version Upgrade Process + +This document outlines the comprehensive process for upgrading Go versions across the Chainloop project. + +NOTE + +- do not upgrade dagger module + +## Overview + +The Chainloop project uses Go in multiple components requiring updates across: +- Source code (go.mod files) +- Docker images +- CI/CD pipelines (GitHub Actions) +- Documentation + +## Step-by-Step Process + +### 1. Get Latest Go Version and Docker Image Digest + +```bash +# Check latest Go version at https://go.dev/doc/devel/release +# Get Docker image SHA256 digest +docker pull golang:1.24.6 +# Note the SHA256 from output: sha256:2c89c41fb9efc3807029b59af69645867cfe978d2b877d475be0d72f6c6ce6f6 +``` + +### 2. Update Source Code + +Update all `go.mod` files in the project: +- `./go.mod` - Main project +- `./extras/dagger/go.mod` - Dagger module + +Change the `go` directive to new version: +```go +go 1.24.6 +``` + +### 3. Update Docker Images + +Update all Dockerfiles with new version and SHA256: +- `./app/artifact-cas/Dockerfile` +- `./app/artifact-cas/Dockerfile.goreleaser` +- `./app/controlplane/Dockerfile` +- `./app/controlplane/Dockerfile.goreleaser` +- `./app/cli/Dockerfile.goreleaser` + +Replace `FROM` lines with: +```dockerfile +FROM golang:1.24.6@sha256:2c89c41fb9efc3807029b59af69645867cfe978d2b877d475be0d72f6c6ce6f6 AS builder +``` + +### 4. Update GitHub Actions + +Update go-version in all workflow files: +- `./.github/workflows/lint.yml` +- `./.github/workflows/test.yml` +- `./.github/workflows/release.yaml` +- `./.github/workflows/codeql.yml` +- `./docs/examples/ci-workflows/github.yaml` + +Find and replace: +```yaml +go-version: "1.24.4" # old version +``` +with: +```yaml +go-version: "1.24.6" # new version +``` + +### 5. Update Documentation + +Update version reference in `./CLAUDE.md` - "Key Technologies" section: +```markdown +- **Language**: Go 1.24.6 +``` + +### 6. Test and Verify + +```bash +make test # Ensure compatibility +make lint # Check code quality +``` + +## Important Notes + +1. **SHA256 Verification**: Always use SHA256 digests for Docker images for security and reproducibility +2. **Test Thoroughly**: Go upgrades can introduce breaking changes +3. **Multiple Components**: CLI, Control Plane, and Artifact CAS all use Go +4. **Dagger Module**: Has separate go.mod that needs updating +5. **Development Environment**: Compose files use pre-built images, don't need Go updates + +## Files Updated in This Process + +### Source Code +- `./go.mod` + +### Docker Images +- `./app/artifact-cas/Dockerfile` +- `./app/artifact-cas/Dockerfile.goreleaser` +- `./app/controlplane/Dockerfile` +- `./app/controlplane/Dockerfile.goreleaser` +- `./app/cli/Dockerfile.goreleaser` + +### CI/CD Workflows +- `./.github/workflows/lint.yml` +- `./.github/workflows/test.yml` +- `./.github/workflows/release.yaml` +- `./.github/workflows/codeql.yml` +- `./docs/examples/ci-workflows/github.yaml` + +### Documentation +- `./CLAUDE.md` \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f27fa6ca2..1fa5804e5 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -48,7 +48,7 @@ jobs: - name: Set up Go uses: actions/setup-go@be3c94b385c4f180051c996d336f57a34c397495 # v3.6.1 with: - go-version: "1.24.4" + go-version: "1.24.6" # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7ce84c793..4b42adcfc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,7 +26,7 @@ jobs: steps: - uses: actions/setup-go@be3c94b385c4f180051c996d336f57a34c397495 # v3.6.1 with: - go-version: "1.24.4" + go-version: "1.24.6" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -67,7 +67,7 @@ jobs: - uses: actions/setup-go@be3c94b385c4f180051c996d336f57a34c397495 # v3.6.1 with: - go-version: "1.24.4" + go-version: "1.24.6" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4faccd312..22832a15b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -78,7 +78,7 @@ jobs: - name: Set up Go uses: actions/setup-go@be3c94b385c4f180051c996d336f57a34c397495 # v3.6.1 with: - go-version: "1.24.4" + go-version: "1.24.6" # install qemu binaries for multiarch builds (needed by goreleaser/buildx) - name: Setup qemu diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5255fc12b..5b1c874c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - uses: actions/setup-go@be3c94b385c4f180051c996d336f57a34c397495 # v3.6.1 with: - go-version: "1.24.4" + go-version: "1.24.6" cache: true cache-dependency-path: go.sum diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..8dc736a3c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,256 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +Chainloop is an open-source evidence store for Software Supply Chain attestations, SBOMs, VEX, SARIF, and other compliance artifacts. The project consists of three main components: a Control Plane (server), Artifact Content Addressable Storage (CAS), and a CLI client. + +## Architecture + +The codebase follows a microservices architecture with three main components and shared libraries: + +### Control Plane (`app/controlplane/`) +Main backend service implementing hexagonal architecture: +- **API Layer** (`./api/`): Protobuf definitions and generated gRPC/HTTP services +- **Server Layer** (`./internal/server`): HTTP/gRPC servers, middlewares, and request handling +- **Service Layer** (`./internal/service`): Protocol buffer service implementations +- **Business Layer** (`./pkg/biz`): Core business logic and repository abstractions + - Organizations, workflows, attestations, API tokens, groups, projects + - CAS backend management, integrations, user access control + - Policy evaluation and referrer management +- **Data Layer** (`./pkg/data`): Repository implementations using Ent ORM + - PostgreSQL with Atlas migrations in `./pkg/data/ent/migrate/migrations/` + - Ent schemas in `./pkg/data/ent/schema/` + +**Dependencies**: OIDC provider, PostgreSQL, Secret storage (Vault/AWS/GCP/Azure), Artifact CAS +**Authentication**: JWT tokens, OIDC delegation, RBAC with Casbin +**Key Features**: Multi-tenancy, workflow contracts, policy as code, audit logging + +### Artifact CAS (`app/artifact-cas/`) +Content Addressable Storage proxy service: +- **API Layer** (`./api/`): gRPC bytestream protocol for efficient streaming +- **Server Layer** (`./internal/server`): gRPC server setup and middlewares +- **Service Layer** (`./internal/service`): Bytestream, download, resource, and status services +- **Storage Backends** (`pkg/blobmanager/`): OCI registry, S3, Azure Blob Storage + - Content-addressable storage with SHA256 digests + - Multi-tenant through runtime credential selection + - Immutable artifact storage with digest verification + +**Dependencies**: Secret storage backend for OCI/blob credentials +**Authentication**: JWT tokens with upload/download permissions from Control Plane +**Key Features**: Multi-backend support, streaming uploads/downloads, content verification + +### CLI (`app/cli/`) +Command-line client for both operators and CI/CD systems: +- **Commands** (`./cmd/`): Cobra-based command structure + - `attestation_*`: Attestation crafting lifecycle (init, add, push, status, verify) + - `workflow_*`: Workflow and contract management + - `organization_*`: Organization and membership operations + - `artifact_*`: Artifact upload/download operations + - `auth_*`: Authentication and account management +- **Actions** (`./internal/action/`): Business logic implementations for each command +- **Policy Development** (`./internal/policydevel/`): Local policy development and testing +- **Telemetry** (`./internal/telemetry/`): Usage analytics with PostHog + +**Key Features**: OIDC authentication, multi-environment config, plugin system, attestation crafting + +### Shared Libraries (`pkg/`) +Common functionality across components: +- **Attestation** (`pkg/attestation/`): + - **Crafter**: Attestation creation with material collection and runner context + - **Materials**: Support for 25+ evidence types (SBOM, SARIF, VEX, OCI images, etc.) + - **Runners**: CI/CD platform integration (GitHub Actions, GitLab, Azure, Jenkins, etc.) + - **Signer**: Chainloop, Cosign, SignServer integration + - **Verifier**: Attestation verification and timestamp validation +- **Policies** (`pkg/policies/`): OPA/Rego policy evaluation engine +- **Blob Manager** (`pkg/blobmanager/`): Multi-backend storage abstraction +- **Credentials** (`pkg/credentials/`): Secret management for multiple providers +- **gRPC Connection** (`pkg/grpcconn/`): Reusable gRPC client setup + +## Development Commands + +### Initial Setup +```bash +make init # Install required development tools +``` + +### Building +```bash +# Root level - build all components +make all # Generate APIs and protobuf bindings +make api # Generate API proto bindings +make generate # Generate code, APIs, configs + +# Individual components +make -C app/controlplane build +make -C app/cli build +make -C app/artifact-cas build +``` + +### Testing +```bash +# Root level +make test # Run all tests + +# Component-specific +make -C app/controlplane test # All tests including integration +make -C app/controlplane test-unit # Unit tests only (SKIP_INTEGRATION=true) +make -C app/cli test +make -C app/artifact-cas test +``` + +### Linting +```bash +# Root level - lint all components +make lint + +# Component-specific +make -C app/controlplane lint # golangci-lint + buf lint +make -C app/cli lint +make -C app/artifact-cas lint +``` + +### Development Server +```bash +# Start auxiliary services (PostgreSQL, Vault, Dex OIDC) +cd devel && docker compose up + +# Run components (in separate terminals) +make -C app/controlplane run # Runs migration_apply first +make -C app/artifact-cas run + +# CLI usage +go run app/cli/main.go --insecure [command] +``` + +### Database Operations +```bash +cd app/controlplane +make migration_apply # Apply migrations to local DB +make migration_sync # Sync migrations with Ent schema +make migration_new # Create empty migration file +make migration_lint # Lint migration files +``` + +## Key Technologies + +- **Language**: Go 1.24.6. To know how to upgrade go version, see @.claude/golang-upgrade-process.md +- **API**: gRPC with HTTP/JSON gateway, Protocol Buffers with buf +- **Database**: PostgreSQL with Ent ORM, Atlas for migrations +- **Authentication**: OIDC, JWT tokens +- **Policy**: Open Policy Agent (OPA) with Rego +- **Storage**: Multi-backend (OCI, S3, Azure Blob, inline) +- **Cryptography**: Sigstore (Cosign, Fulcio), in-toto attestations +- **Standards**: SLSA, SBOM (SPDX/CycloneDX), SARIF, OpenVEX, CSAF + +## Development Environment + +### Setup Options + +**Option 1: Local Development (Recommended)** +```bash +# 1. Install tools +make init + +# 2. Start auxiliary services +cd devel && docker compose up + +# 3. Run components locally (separate terminals) +make -C app/controlplane run +make -C app/artifact-cas run + +# 4. Configure CLI +go run app/cli/main.go config save --insecure --control-plane localhost:9000 --artifact-cas localhost:9001 +go run app/cli/main.go --insecure auth login +``` + +**Option 2: Containerized Labs Environment** +```bash +# 1. Add dex hostname to /etc/hosts +echo "127.0.0.1 dex" | sudo tee -a /etc/hosts + +# 2. Run full containerized stack +docker compose -f devel/compose.labs.yml up + +# 3. Extract development token from logs +docker compose -f devel/compose.labs.yml logs control-plane | grep "DEVELOPMENT USER TOKEN" -A 1 + +# 4. Authenticate with token +chainloop --insecure auth login --skip-browser +``` + +### Development Infrastructure + +**Compose Configuration**: +- `compose.common.yml`: PostgreSQL 16 + Vault (in-memory dev mode) +- `compose.yml`: Adds Dex OIDC for local development +- `compose.labs.yml`: Full containerized environment with pre-built images + +**Development Services**: +- **PostgreSQL**: `localhost:5432`, database: `controlplane`, user: `postgres`, no password +- **Vault**: `localhost:8200`, dev token: `notasecret` (in-memory, data lost on restart) +- **Dex OIDC**: `localhost:5556`, static users with password `"password"` +- **Control Plane**: `localhost:9000` (gRPC) / `localhost:8000` (HTTP) +- **Artifact CAS**: `localhost:9001` (gRPC) / `localhost:8001` (HTTP) +- **Optional Minio S3**: `localhost:9002` (API) / `localhost:9003` (Console) with `--profile optional` + +**Development Credentials**: +- **OIDC Users**: `sarah@chainloop.local` / `john@chainloop.local` (password: `password`) +- **Signing Keys**: Development keypairs in `devel/devkeys/` (DO NOT USE IN PRODUCTION) + - `ca.pem`/`ca.pub`: Certificate Authority + - `cas.pem`/`cas.pub`: Artifact CAS signing + - `freetsa.pem`: Free TSA timestamp authority + - Self-signed certificates in `devkeys/selfsigned/` + +## Testing Environment + +- **Integration Tests**: Use testcontainers for isolated database testing +- **Unit Tests**: Run with `make test-unit` or `SKIP_INTEGRATION=true make test` +- **Local PostgreSQL**: `postgres://postgres:@localhost:5432/controlplane?sslmode=disable` +- **macOS Docker Fix**: `sudo ln -s $HOME/.docker/run/docker.sock /var/run/docker.sock` + +## Code Generation + +The project heavily uses code generation: +- **Protobuf**: API definitions and gRPC services +- **Wire**: Dependency injection +- **Ent**: ORM models and queries +- **Buf**: Protobuf tooling and validation + +Always run `make generate` after modifying .proto files or Ent schemas. + +## Contract-Based Development + +Workflow Contracts define the structure and requirements for CI/CD attestations. They specify what materials must be collected and policies that must be evaluated. + +## Component-Specific Development + +### Control Plane Development +- **Schema Changes**: Update Ent schemas in `pkg/data/ent/schema/`, then run `make generate && make migration_new && make migration_apply` +- **API Changes**: Modify `.proto` files, then run `make api` to regenerate code +- **Business Logic**: Implement in `pkg/biz/` layer with repository interfaces +- **Tests**: Unit tests with `make test-unit`, integration tests with `make test` (uses testcontainers) + +### CLI Development +- **Commands**: Add new commands in `cmd/` following Cobra patterns +- **Actions**: Implement business logic in `internal/action/` +- **Policy Development**: Use `internal/policydevel/` for local policy testing +- **Default Endpoints**: Override with `-ldflags` at build time using `defaultCASAPI` and `defaultCPAPI` variables + +### Artifact CAS Development +- **Storage Backends**: Implement new backends in `pkg/blobmanager/` with Provider interface +- **Authentication**: JWT tokens generated by Control Plane, validated by CAS +- **Streaming**: Uses gRPC bytestream protocol for efficient file transfers + +## Commit Guidelines + +All commits must meet these criteria: +- **Signed**: Use `-S` flag ([signing guide](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits)) +- **Developer Certificate of Origin**: Use `--sign-off` flag +- **Conventional Commits**: Follow [format guidelines](https://www.conventionalcommits.org/en/v1.0.0) +- **Example**: `git commit -S -s -m "feat: add new material type"` + +Code reviews are required for all submissions via GitHub pull requests. +- make sure golang code is always formatted and golang-ci-lint is run + diff --git a/app/artifact-cas/Dockerfile b/app/artifact-cas/Dockerfile index dd490fb71..0d6243a05 100644 --- a/app/artifact-cas/Dockerfile +++ b/app/artifact-cas/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24.4@sha256:be70d93633d07a2acae4ff3401672b04f23e5850b0248d65c23e30dc75dded09 AS builder +FROM golang:1.24.6@sha256:2c89c41fb9efc3807029b59af69645867cfe978d2b877d475be0d72f6c6ce6f6 AS builder # Not linked libraries since it will be injected into a scratch container ENV CGO_ENABLED=0 diff --git a/app/artifact-cas/Dockerfile.goreleaser b/app/artifact-cas/Dockerfile.goreleaser index ba9e84804..89dfcec3e 100644 --- a/app/artifact-cas/Dockerfile.goreleaser +++ b/app/artifact-cas/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM golang:1.24.4@sha256:be70d93633d07a2acae4ff3401672b04f23e5850b0248d65c23e30dc75dded09 AS builder +FROM golang:1.24.6@sha256:2c89c41fb9efc3807029b59af69645867cfe978d2b877d475be0d72f6c6ce6f6 AS builder FROM scratch diff --git a/app/cli/Dockerfile.goreleaser b/app/cli/Dockerfile.goreleaser index 358281241..e7f0bbd11 100644 --- a/app/cli/Dockerfile.goreleaser +++ b/app/cli/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM golang:1.24.4@sha256:be70d93633d07a2acae4ff3401672b04f23e5850b0248d65c23e30dc75dded09 AS builder +FROM golang:1.24.6@sha256:2c89c41fb9efc3807029b59af69645867cfe978d2b877d475be0d72f6c6ce6f6 AS builder RUN mkdir -p /.config/chainloop FROM scratch diff --git a/app/controlplane/Dockerfile b/app/controlplane/Dockerfile index 4af251cae..89dc1ab15 100644 --- a/app/controlplane/Dockerfile +++ b/app/controlplane/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24.4@sha256:be70d93633d07a2acae4ff3401672b04f23e5850b0248d65c23e30dc75dded09 AS builder +FROM golang:1.24.6@sha256:2c89c41fb9efc3807029b59af69645867cfe978d2b877d475be0d72f6c6ce6f6 AS builder # Not linked libraries since it will be injected into a scratch container ENV CGO_ENABLED=0 diff --git a/app/controlplane/Dockerfile.goreleaser b/app/controlplane/Dockerfile.goreleaser index 11b1865f4..9d14c9c81 100644 --- a/app/controlplane/Dockerfile.goreleaser +++ b/app/controlplane/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM golang:1.24.4@sha256:be70d93633d07a2acae4ff3401672b04f23e5850b0248d65c23e30dc75dded09 AS builder +FROM golang:1.24.6@sha256:2c89c41fb9efc3807029b59af69645867cfe978d2b877d475be0d72f6c6ce6f6 AS builder FROM scratch diff --git a/docs/examples/ci-workflows/github.yaml b/docs/examples/ci-workflows/github.yaml index a6837a6c1..7343231ed 100644 --- a/docs/examples/ci-workflows/github.yaml +++ b/docs/examples/ci-workflows/github.yaml @@ -41,7 +41,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: "1.24.4" + go-version: "1.24.6" # Generate SBOM using syft in cycloneDX format - uses: anchore/sbom-action@v0 diff --git a/go.mod b/go.mod index 24e46ccaa..027cd401b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/chainloop-dev/chainloop -go 1.24.4 +go 1.24.6 require ( cloud.google.com/go/secretmanager v1.14.2