From 4d305d52ead17ba60d664e012fdbbee329d75489 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Wed, 1 Jul 2026 09:44:03 -0700 Subject: [PATCH 1/9] Add mTLS between atenet router and ate apiserver Rebased onto main and integrated with the ateapi-auth (mtls|jwt) mode that landed on main in #248. Router (client): - ateapi-auth=mtls now performs real mutual TLS: the router presents its podidentity credential bundle (--ateapi-client-cert) as a client cert and verifies ateapi's serving cert against the servicedns trust bundle (--ateapi-ca-certs), instead of the previous insecure TLS dial. No insecure fallback: missing material is a hard error. - ateapi-auth=jwt (from #248) is unchanged and still routes through ateapiauth.DialOptions; otel tracing/metrics are preserved. ateapi (server): - The ateapiauth server interceptor now injects an authenticated principal (internal/principal) into the request context in both modes: the mtls authenticator extracts the client's SPIFFE identity from its mTLS peer cert, and the jwt authenticator records the verified token subject (VerifyBearerToken now returns the principal ID). The principal carries a Kind (mtls|jwt) and is included in RPC request logs. Supporting changes: - Move credbundle to internal/credbundle and add ClientLoader for client certs. - podcertcontroller signers, valkey, and install-ate.sh updated to provision and mount the podidentity/servicedns bundles; manifests mount them into the router and apiserver pods. --- cmd/ateapi/main.go | 11 ++- cmd/atenet/internal/router.go | 4 +- cmd/atenet/internal/router/router.go | 85 +++++++++++++++-- cmd/atenet/internal/router/status_test.go | 71 ++++++++++++-- .../podidentitysigner/podidentitysigner.go | 8 +- .../servicednssigner/servicednssigner.go | 13 ++- hack/install-ate.sh | 39 ++++++-- internal/ateapiauth/server.go | 67 +++++++++++--- internal/ateapiauth/server_test.go | 92 +++++++++++++++++-- internal/ateinterceptors/ateinterceptors.go | 4 + .../credbundle/credbundle.go | 11 +++ .../credbundle/credbundle_test.go | 0 internal/principal/principal.go | 45 +++++++++ manifests/ate-install/ate-api-server.yaml | 11 +++ manifests/ate-install/atenet-router.yaml | 25 +++++ manifests/ate-install/valkey.yaml | 30 ++++-- 16 files changed, 457 insertions(+), 59 deletions(-) rename {cmd/ateapi/internal => internal}/credbundle/credbundle.go (85%) rename {cmd/ateapi/internal => internal}/credbundle/credbundle_test.go (100%) create mode 100644 internal/principal/principal.go diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index dd481bc50..4bd8faf07 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -28,13 +28,13 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/credbundle" "github.com/agent-substrate/substrate/cmd/ateapi/internal/k8sjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/sessionidentity" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -168,9 +168,12 @@ func main() { authCfg := ateapiauth.ServerConfig{ Mode: authModeParsed, - VerifyBearerToken: func(ctx context.Context, bearer string) error { - _, err := k8sjwt.Verify(ctx, jwtIssuerDiscoveryClient, bearer, *clientJWTIssuer, *clientJWTAudience, time.Now()) - return err + VerifyBearerToken: func(ctx context.Context, bearer string) (string, error) { + claims, err := k8sjwt.Verify(ctx, jwtIssuerDiscoveryClient, bearer, *clientJWTIssuer, *clientJWTAudience, time.Now()) + if err != nil { + return "", err + } + return claims.Subject, nil }, } if err := ateapiauth.ValidateServerConfig(authCfg); err != nil { diff --git a/cmd/atenet/internal/router.go b/cmd/atenet/internal/router.go index c58f31a7e..d62962a44 100644 --- a/cmd/atenet/internal/router.go +++ b/cmd/atenet/internal/router.go @@ -58,7 +58,9 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)") cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)") - cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.") + cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) presents the podidentity client cert (--ateapi-client-cert) and verifies ateapi's serving cert against the servicedns trust bundle (--ateapi-ca-certs). 'jwt' verifies the server cert and sends a Bearer SA token.") + cmd.Flags().StringVar(&cfg.AteapiClientCertPath, "ateapi-client-cert", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Path to the podidentity credential bundle the router presents as its client cert to ateapi (mtls mode).") + cmd.Flags().StringVar(&cfg.AteapiCACertsPath, "ateapi-ca-certs", "/run/servicedns-ca.podcert.ate.dev/trust-bundle.pem", "Path to the servicedns trust bundle used to verify ateapi's serving cert (mtls mode).") cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") cmd.Flags().StringVar(&cfg.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") cmd.Flags().StringVar(&cfg.AteapiTokenFile, "ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 7ed77df73..7f18045b8 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -18,6 +18,7 @@ import ( "context" "crypto/rand" "crypto/rsa" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -39,6 +40,7 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/sync/errgroup" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" @@ -48,6 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/agent-substrate/substrate/internal/ateapiauth" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/serverboot" v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -88,6 +91,13 @@ type RouterConfig struct { AteapiCAFile string AteapiServerName string AteapiTokenFile string + + // mTLS mode (ateapi-auth=mtls) material: the router presents + // AteapiClientCertPath (the podidentity credential bundle) as its client + // cert and verifies ateapi's serving cert against AteapiCACertsPath (the + // servicedns trust bundle). + AteapiClientCertPath string + AteapiCACertsPath string } // RouterServer instantiates and coordinates runtime threads executing system modules. @@ -149,6 +159,53 @@ func NewRouterServer(cfg RouterConfig) (*RouterServer, error) { }, nil } +// ateapiTransportCreds builds the mTLS credentials the router uses to dial +// ateapi in ateapi-auth=mtls mode. When both the servicedns trust bundle and +// the podidentity client credential bundle are present (the in-cluster case, +// mounted via projected pod-certificate volumes), it verifies ateapi's serving +// cert against the servicedns trust bundle and presents its own podidentity +// SPIFFE client cert. When that material is absent, it returns an error rather +// than falling back to an insecure connection. +func ateapiTransportCreds(cfg RouterConfig) (credentials.TransportCredentials, error) { + tlsCfg, err := ateapiTLSConfig(cfg) + if err != nil { + return nil, err + } + return credentials.NewTLS(tlsCfg), nil +} + +func ateapiTLSConfig(cfg RouterConfig) (*tls.Config, error) { + haveCA := fileExists(cfg.AteapiCACertsPath) + haveClientCert := fileExists(cfg.AteapiClientCertPath) + + if !haveCA || !haveClientCert { + return nil, fmt.Errorf("ateapi mTLS material not found: ca-certs=%q client-cert=%q", + cfg.AteapiCACertsPath, cfg.AteapiClientCertPath) + } + + caBytes, err := os.ReadFile(cfg.AteapiCACertsPath) + if err != nil { + return nil, fmt.Errorf("read ateapi CA certs: %w", err) + } + rootCAs := x509.NewCertPool() + if !rootCAs.AppendCertsFromPEM(caBytes) { + return nil, fmt.Errorf("parse ateapi CA certs from %s", cfg.AteapiCACertsPath) + } + + return &tls.Config{ + RootCAs: rootCAs, + GetClientCertificate: credbundle.ClientLoader(cfg.AteapiClientCertPath), + }, nil +} + +func fileExists(path string) bool { + if path == "" { + return false + } + _, err := os.Stat(path) + return err == nil +} + func (s *RouterServer) Run(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -197,14 +254,26 @@ func (s *RouterServer) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("invalid --ateapi-auth: %w", err) } - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: authMode, - CAFile: s.cfg.AteapiCAFile, - ServerName: s.cfg.AteapiServerName, - TokenFile: s.cfg.AteapiTokenFile, - }) - if err != nil { - return fmt.Errorf("building ateapi dial options: %w", err) + var dialOpts []grpc.DialOption + switch authMode { + case ateapiauth.ModeMTLS: + // Real mutual TLS: present the router's podidentity client cert and + // verify ateapi's serving cert against the servicedns trust bundle. + creds, err := ateapiTransportCreds(s.cfg) + if err != nil { + return fmt.Errorf("building ateapi mTLS credentials: %w", err) + } + dialOpts = []grpc.DialOption{grpc.WithTransportCredentials(creds)} + default: + dialOpts, err = ateapiauth.DialOptions(ateapiauth.ClientConfig{ + Mode: authMode, + CAFile: s.cfg.AteapiCAFile, + ServerName: s.cfg.AteapiServerName, + TokenFile: s.cfg.AteapiTokenFile, + }) + if err != nil { + return fmt.Errorf("building ateapi dial options: %w", err) + } } dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) conn, err := grpc.NewClient( diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index a72011848..64d003ca8 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -16,12 +16,20 @@ package router import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" "io" + "math/big" "net" "net/http" "os" + "path/filepath" "strings" "testing" "time" @@ -47,16 +55,20 @@ func TestStatuszEndpoint(t *testing.T) { defer os.Remove(tmpFile.Name()) tmpFile.Close() + // Run() dials ateapi in mtls mode, which requires real TLS material; generate it. + caPath, clientCertPath := writeTestTLSMaterial(t) + cfg := RouterConfig{ - Standalone: true, - Namespace: "default", - StatusPort: httpPort, - HttpPort: 8080, - XdsPort: 18000, - ExtprocPort: 50051, - TemplatesFile: tmpFile.Name(), - MetricsAddr: "127.0.0.1:0", - AteapiCAFile: "unused-in-mtls-mode", + Standalone: true, + Namespace: "default", + StatusPort: httpPort, + HttpPort: 8080, + XdsPort: 18000, + ExtprocPort: 50051, + TemplatesFile: tmpFile.Name(), + MetricsAddr: "127.0.0.1:0", + AteapiCACertsPath: caPath, + AteapiClientCertPath: clientCertPath, } srv, err := NewRouterServer(cfg) @@ -142,3 +154,44 @@ func TestStatuszEndpoint(t *testing.T) { t.Errorf("Target parameters unassigned inside context payload context properties: found %s", dashboard.Queries[0].Target) } } + +// writeTestTLSMaterial generates a self-signed certificate and writes a CA trust +// bundle and a client credential bundle to temp files, returning their paths. +// Run() (mtls mode) requires both to build its ateapi mTLS credentials. +func writeTestTLSMaterial(t *testing.T) (caPath, clientCertPath string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating certificate: %v", err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshaling key: %v", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + + dir := t.TempDir() + caPath = filepath.Join(dir, "ca.pem") + if err := os.WriteFile(caPath, certPEM, 0o600); err != nil { + t.Fatalf("writing CA file: %v", err) + } + clientCertPath = filepath.Join(dir, "client.pem") + if err := os.WriteFile(clientCertPath, append(certPEM, keyPEM...), 0o600); err != nil { + t.Fatalf("writing client cert file: %v", err) + } + return caPath, clientCertPath +} diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go index 5b63a15e4..6d23da194 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go @@ -121,6 +121,7 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq Path: path.Join("ns", pcr.ObjectMeta.Namespace, "sa", pcr.Spec.ServiceAccountName), } + parent := h.caPool.CAs[0].RootCertificate template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, @@ -128,9 +129,14 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq URIs: []*url.URL{spiffeURI}, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + // Link the leaf to its issuing CA by key id so verifiers can disambiguate + // a multi-CA trust bundle (e.g. valkey trusts both the servicedns and + // podidentity CAs). + // https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.1 + AuthorityKeyId: parent.SubjectKeyId, } - subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, h.caPool.CAs[0].RootCertificate, subjectPublicKey, h.caPool.CAs[0].SigningKey) + subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) if err != nil { return fmt.Errorf("while signing subject cert: %w", err) } diff --git a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go index 4258f1921..1a995b20d 100644 --- a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go +++ b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go @@ -135,6 +135,13 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq } } + // This is returned as a transient error to allow retries. + // Without this, ate-apiserver can have a servicedns cert without DNS name + // while the covering Service is being created, and cache it for 24 hours. + if len(dnsNames) == 0 { + return fmt.Errorf("pod %s/%s is not (yet) selected by any Service; refusing to issue a serving cert with no DNS SANs", pcr.ObjectMeta.Namespace, pcr.Spec.PodName) + } + // TODO: Encode the OIDC issuer of the cluster into the certificate. subjectPublicKey, err := podcertificate.PublicKey(pcr) @@ -156,6 +163,7 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq notAfter := notBefore.Add(lifetime) beginRefreshAt := notAfter.Add(-30 * time.Minute) + parent := h.caPool.CAs[0].RootCertificate template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, @@ -163,9 +171,12 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq DNSNames: dnsNames, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + // Link the leaf to its issuing CA by key id. Needed this for Valkey + // to understand which CA to use when validating a client cert. + AuthorityKeyId: parent.SubjectKeyId, } - subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, h.caPool.CAs[0].RootCertificate, subjectPublicKey, h.caPool.CAs[0].SigningKey) + subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) if err != nil { return fmt.Errorf("while signing subject cert: %w", err) } diff --git a/hack/install-ate.sh b/hack/install-ate.sh index e7e910a24..658ccb2d2 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -167,17 +167,36 @@ render_ate_system_manifests() { fi } -create_valkey_ca_certs_secret() { - log_step "create_valkey_ca_certs_secret" - local ca_certs="" - # Extract from in-cluster service-dns-ca-pool secret (base64 json) +# Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate. +ca_pool_root_pem() { + local secret="$1" local pool_json="" - pool_json=$(run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool -o jsonpath='{.data.pool}' | base64 --decode) - # Extract RootCertificateDER base64 string + pool_json=$(run_kubectl get secret -n podcertificate-controller-system "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) local der_base64="" der_base64=$(echo "${pool_json}" | grep -o '"RootCertificateDER":"[^"]*' | sed 's/"RootCertificateDER":"//') - # Convert DER to PEM certificate - ca_certs=$(echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem) + echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem +} + +create_valkey_ca_certs_secret() { + log_step "create_valkey_ca_certs_secret" + # valkey requires a single tls-ca-cert-file to verify client and server certs it sees, + # so it needs both CAs: + # - servicedns CA: verifies valkey peers' server certs. + # - podidentity CA: verifies the client certs that connect to valkey + # (apiserver, the init job, and peers acting as clients). + # Extract each root into its own variable: errexit cannot see a substitution + # failing inside printf's argument list, which would silently produce a CA + # file with a missing root. + local servicedns_root="" + servicedns_root=$(ca_pool_root_pem service-dns-ca-pool) + local podidentity_root="" + podidentity_root=$(ca_pool_root_pem pod-identity-ca-pool) + if [[ -z "${servicedns_root}" || -z "${podidentity_root}" ]]; then + echo "error: failed to extract a CA root for valkey-ca-certs" >&2 + return 1 + fi + local ca_certs="" + ca_certs=$(printf '%s\n%s\n' "${servicedns_root}" "${podidentity_root}") run_kubectl create secret generic valkey-ca-certs \ --from-literal=ca.crt="${ca_certs}" \ @@ -227,7 +246,9 @@ create_api_server_env_vars() { redis_address="valkey-cluster.ate-system.svc:6379" use_iam_auth="false" tls_server_name="valkey-cluster.ate-system.svc" - client_cert="/run/servicedns.podcert.ate.dev/credential-bundle.pem" + # The apiserver dials valkey as a client, so it presents a podidentity + # (SPIFFE) client cert rather than a servicedns serving cert. + client_cert="/run/podidentity.podcert.ate.dev/credential-bundle.pem" echo "REDIS_ADDRESS: ${redis_address}" diff --git a/internal/ateapiauth/server.go b/internal/ateapiauth/server.go index b8e5167bf..3baaf476f 100644 --- a/internal/ateapiauth/server.go +++ b/internal/ateapiauth/server.go @@ -24,11 +24,15 @@ package ateapiauth import ( "context" "fmt" + "log/slog" "strings" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" ) @@ -73,9 +77,10 @@ func ValidateServerConfig(cfg ServerConfig) error { type ServerConfig struct { Mode Mode - // VerifyBearerToken verifies a Bearer token presented by a client. Required - // for ModeJWT and ignored for ModeMTLS. - VerifyBearerToken func(context.Context, string) error + // VerifyBearerToken verifies a Bearer token presented by a client and + // returns the authenticated principal's ID (e.g. the JWT subject). + // Required for ModeJWT and ignored for ModeMTLS. + VerifyBearerToken func(context.Context, string) (string, error) } // UnaryServerInterceptor returns a gRPC unary interceptor enforcing cfg. @@ -129,13 +134,51 @@ func serverAuthenticatorFor(cfg ServerConfig) serverAuthenticator { type mtlsServerAuthenticator struct{} func (mtlsServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { - // TODO: Extract the transport-authenticated client identity and attach it - // to ctx once ateapi has an authorization layer. - return ctx, nil + pInfo := principal.PrincipalInfo{ + ID: "anonymous", + } + if id, ok := mtlsPeerIdentity(ctx); ok { + pInfo = principal.PrincipalInfo{ + ID: id, + Kind: principal.KindMTLS, + } + } + return principal.InjectContext(ctx, pInfo), nil +} + +// mtlsPeerIdentity extracts the client identity (the first URI SAN, a SPIFFE +// ID) from the transport-authenticated peer certificate, if any. +func mtlsPeerIdentity(ctx context.Context) (string, bool) { + p, ok := peer.FromContext(ctx) + if !ok || p.AuthInfo == nil { + slog.ErrorContext(ctx, "Authentication failed: no peer or auth info in context.") + return "", false + } + + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok { + slog.ErrorContext(ctx, "Authentication failed: no TLS info in context.") + return "", false + } + + if len(tlsInfo.State.PeerCertificates) == 0 { + slog.ErrorContext(ctx, "Authentication failed: no peer certificates in TLS info.") + return "", false + } + + clientCert := tlsInfo.State.PeerCertificates[0] + if len(clientCert.URIs) == 0 { + slog.ErrorContext(ctx, "Authentication failed: no URIs in peer certificate.") + return "", false + } + + id := clientCert.URIs[0].String() + slog.InfoContext(ctx, "Authentication successful", slog.String("id", id)) + return id, true } type jwtServerAuthenticator struct { - verifyBearerToken func(context.Context, string) error + verifyBearerToken func(context.Context, string) (string, error) } func (a jwtServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { @@ -143,12 +186,14 @@ func (a jwtServerAuthenticator) authenticate(ctx context.Context) (context.Conte if !ok { return nil, status.Error(codes.Unauthenticated, "missing bearer token") } - if err := a.verifyBearerToken(ctx, bearer); err != nil { + id, err := a.verifyBearerToken(ctx, bearer) + if err != nil { return nil, status.Errorf(codes.Unauthenticated, "invalid bearer token: %v", err) } - // TODO: Attach the verified JWT identity to ctx once ateapi has an - // authorization layer that consumes it. - return ctx, nil + return principal.InjectContext(ctx, principal.PrincipalInfo{ + ID: id, + Kind: principal.KindJWT, + }), nil } type invalidServerAuthenticator struct { diff --git a/internal/ateapiauth/server_test.go b/internal/ateapiauth/server_test.go index 5a0c0631e..b71e855b4 100644 --- a/internal/ateapiauth/server_test.go +++ b/internal/ateapiauth/server_test.go @@ -16,12 +16,18 @@ package ateapiauth import ( "context" + "crypto/tls" + "crypto/x509" "fmt" + "net/url" "testing" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" ) @@ -56,7 +62,7 @@ func TestValidateServerConfig(t *testing.T) { }{ {name: "mtls zero config", cfg: ServerConfig{Mode: ModeMTLS}}, {name: "empty mode zero config", cfg: ServerConfig{}}, - {name: "jwt valid", cfg: ServerConfig{Mode: ModeJWT, VerifyBearerToken: func(context.Context, string) error { return nil }}}, + {name: "jwt valid", cfg: ServerConfig{Mode: ModeJWT, VerifyBearerToken: func(context.Context, string) (string, error) { return "", nil }}}, {name: "jwt missing verifier", cfg: ServerConfig{Mode: ModeJWT}, wantErr: true}, {name: "unknown mode", cfg: ServerConfig{Mode: Mode("bogus")}, wantErr: true}, } @@ -71,17 +77,66 @@ func TestValidateServerConfig(t *testing.T) { } } -func TestMTLSServerAuthenticatorAllowsAnonymous(t *testing.T) { - _, err := (mtlsServerAuthenticator{}).authenticate(context.Background()) - if err != nil { - t.Fatalf("ModeMTLS should not error: %v", err) +func TestMTLSServerAuthenticatorPrincipal(t *testing.T) { + spiffeID := &url.URL{Scheme: "spiffe", Host: "ate.dev", Path: "/ns/default/sa/router"} + tests := []struct { + name string + ctx context.Context + want principal.PrincipalInfo + }{ + { + name: "no peer", + ctx: context.Background(), + want: principal.PrincipalInfo{ID: "anonymous"}, + }, + { + name: "peer without certificates", + ctx: peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{}, + }), + want: principal.PrincipalInfo{ID: "anonymous"}, + }, + { + name: "certificate without URI SAN", + ctx: peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{}}, + }}, + }), + want: principal.PrincipalInfo{ID: "anonymous"}, + }, + { + name: "certificate with SPIFFE URI SAN", + ctx: peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{spiffeID}}}, + }}, + }), + want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newCtx, err := (mtlsServerAuthenticator{}).authenticate(tt.ctx) + if err != nil { + t.Fatalf("ModeMTLS should not error: %v", err) + } + got, ok := principal.FromContext(newCtx) + if !ok { + t.Fatal("no principal in context") + } + if got != tt.want { + t.Errorf("principal=%+v want %+v", got, tt.want) + } + }) } } func TestJWTServerAuthenticatorRequiresBearer(t *testing.T) { auth := jwtServerAuthenticator{ - verifyBearerToken: func(context.Context, string) error { - return fmt.Errorf("bad token") + verifyBearerToken: func(context.Context, string) (string, error) { + return "", fmt.Errorf("bad token") }, } @@ -99,6 +154,29 @@ func TestJWTServerAuthenticatorRequiresBearer(t *testing.T) { } } +func TestJWTServerAuthenticatorInjectsPrincipal(t *testing.T) { + const subject = "system:serviceaccount:default:router" + auth := jwtServerAuthenticator{ + verifyBearerToken: func(context.Context, string) (string, error) { + return subject, nil + }, + } + + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer good-token")) + newCtx, err := auth.authenticate(ctx) + if err != nil { + t.Fatalf("authenticate: %v", err) + } + got, ok := principal.FromContext(newCtx) + if !ok { + t.Fatal("no principal in context") + } + want := principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT} + if got != want { + t.Errorf("principal=%+v want %+v", got, want) + } +} + func TestBearerToken(t *testing.T) { cases := []struct { name string diff --git a/internal/ateinterceptors/ateinterceptors.go b/internal/ateinterceptors/ateinterceptors.go index 2dfeceaec..55e599248 100644 --- a/internal/ateinterceptors/ateinterceptors.go +++ b/internal/ateinterceptors/ateinterceptors.go @@ -21,6 +21,7 @@ import ( "strconv" "time" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -46,12 +47,15 @@ func ServerUnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServer strconv.FormatInt(elapsed.Microseconds(), 10), )) + pInfo, _ := principal.FromContext(ctx) + slog.InfoContext(ctx, "Handle RPC", slog.String("method", info.FullMethod), slog.Any("req", sanitizeForLog(req)), slog.Any("resp", sanitizeForLog(resp)), slog.Any("err", err), slog.String("elapsed-time", elapsed.String()), + slog.Any("principal", pInfo), ) if err != nil { diff --git a/cmd/ateapi/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go similarity index 85% rename from cmd/ateapi/internal/credbundle/credbundle.go rename to internal/credbundle/credbundle.go index 58d86df76..659a53899 100644 --- a/cmd/ateapi/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -38,6 +38,17 @@ func Loader(path string) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { } } +// ClientLoader is the client-side counterpart to Loader. It returns a function +// suitable for use as GetClientCertificate in a tls.Config, re-reading the +// bundle on each handshake so that in-place pod-certificate rotations are +// picked up. +func ClientLoader(path string) func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + // TODO: Introduce caching. + return func(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return Parse(path) + } +} + // Parse reads a private key and certificate chain from a credential bundle file as written by the // Kubernetes Pod Certificates mechanism. func Parse(bundlePath string) (*tls.Certificate, error) { diff --git a/cmd/ateapi/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go similarity index 100% rename from cmd/ateapi/internal/credbundle/credbundle_test.go rename to internal/credbundle/credbundle_test.go diff --git a/internal/principal/principal.go b/internal/principal/principal.go new file mode 100644 index 000000000..c6169aaed --- /dev/null +++ b/internal/principal/principal.go @@ -0,0 +1,45 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package principal + +import "context" + +// Kind values for PrincipalInfo, named after the authentication method that +// established the identity. +const ( + KindMTLS = "mtls" + KindJWT = "jwt" +) + +// PrincipalInfo contains information about an authenticated principal. +type PrincipalInfo struct { + ID string + Kind string +} + +type contextKey struct{} + +var principalKey = contextKey{} + +// FromContext returns the PrincipalInfo from the context, if any. +func FromContext(ctx context.Context) (PrincipalInfo, bool) { + p, ok := ctx.Value(principalKey).(PrincipalInfo) + return p, ok +} + +// InjectContext returns a new context with the given PrincipalInfo. +func InjectContext(ctx context.Context, p PrincipalInfo) context.Context { + return context.WithValue(ctx, principalKey, p) +} diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 0b7b0c0ee..eec53340b 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -117,8 +117,12 @@ spec: name: ate-api-server-envvars optional: true volumeMounts: + # servicedns: the apiserver's own gRPC serving cert (DNS SAN api.ate-system.svc). - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" + # podidentity: the apiserver's client identity (SPIFFE) when it dials valkey. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" - name: "session-id-jwt-pool" mountPath: "/run/session-id-jwt-pool" # Note: See README.md for how to generate this secret. @@ -149,6 +153,13 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem - name: "session-id-jwt-pool" projected: sources: diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 967f22c29..dfc109087 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -132,6 +132,8 @@ spec: - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - "--ateapi-address=api.ate-system.svc:443" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--ateapi-ca-certs=/run/servicedns-ca.podcert.ate.dev/trust-bundle.pem" - "--status-port=4040" - "--port-https=8443" - "--envoy-cert-path=/run/servicedns.podcert.ate.dev/credential-bundle.pem" @@ -163,6 +165,13 @@ spec: containerPort: 4040 - name: metrics containerPort: 9090 + volumeMounts: + # Router's own client identity presented to ateapi. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: "servicedns-ca" + mountPath: "/run/servicedns-ca.podcert.ate.dev" - name: envoy image: envoyproxy/envoy:v1.30-latest command: @@ -194,6 +203,22 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem --- apiVersion: v1 kind: Service diff --git a/manifests/ate-install/valkey.yaml b/manifests/ate-install/valkey.yaml index 2e5fb0088..435352106 100644 --- a/manifests/ate-install/valkey.yaml +++ b/manifests/ate-install/valkey.yaml @@ -28,6 +28,8 @@ data: # Load certificates from projected volume tls-cert-file /run/servicedns.podcert.ate.dev/credential-bundle.pem tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem + tls-client-cert-file /run/podidentity.podcert.ate.dev/credential-bundle.pem + tls-client-key-file /run/podidentity.podcert.ate.dev/credential-bundle.pem tls-ca-cert-file /etc/valkey-ca/ca.crt tls-auth-clients yes # Reload the cert every 12h. @@ -114,6 +116,8 @@ spec: mountPath: /etc/valkey - name: servicedns mountPath: /run/servicedns.podcert.ate.dev + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev - name: valkey-ca-certs mountPath: /etc/valkey-ca readOnly: true @@ -130,6 +134,13 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem - name: valkey-ca-certs projected: sources: @@ -163,8 +174,11 @@ spec: - name: init image: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 volumeMounts: - - name: servicedns - mountPath: /run/servicedns.podcert.ate.dev + # The init job is a client of the cluster, not a Service-backed server, + # so it uses a podidentity client cert rather than a servicedns + # serving cert (which requires DNS SANs it would not have). + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev - name: valkey-ca-certs mountPath: /etc/valkey-ca readOnly: true @@ -188,19 +202,19 @@ spec: done echo "Checking if Valkey cluster is already initialized..." - until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc ping >/dev/null 2>&1; do + until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc ping >/dev/null 2>&1; do echo "Waiting for valkey-cluster-0 to respond to ping..." sleep 2 done - INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc cluster info 2>/dev/null | grep cluster_state || true) + INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc cluster info 2>/dev/null | grep cluster_state || true) if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then echo "Initializing Valkey cluster..." valkey-cli --tls \ --cacert /etc/valkey-ca/ca.crt \ - --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem \ - --key /run/servicedns.podcert.ate.dev/credential-bundle.pem \ + --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ --cluster create ${VALKEY_NODES} \ --cluster-replicas 1 \ --cluster-yes @@ -209,11 +223,11 @@ spec: echo "Cluster already initialized." fi volumes: - - name: servicedns + - name: podidentity projected: sources: - podCertificate: - signerName: servicedns.podcert.ate.dev/identity + signerName: podidentity.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem - name: valkey-ca-certs From 57707991a6c067ae4bf229339c5783f0eefca034 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Sat, 20 Jun 2026 16:57:28 -0700 Subject: [PATCH 2/9] Add ate-apiserver to atelet mTLS --- cmd/ateapi/internal/controlapi/dialer.go | 110 ++++++++++++- cmd/ateapi/internal/controlapi/dialer_test.go | 152 ++++++++++++++++++ .../internal/controlapi/functional_test.go | 9 +- cmd/ateapi/main.go | 9 +- cmd/atelet/main.go | 38 ++++- .../podidentitysigner/podidentitysigner.go | 23 ++- manifests/ate-install/ate-api-server.yaml | 10 +- manifests/ate-install/atelet.yaml | 21 +++ .../kind/atelet/kustomization.yaml | 2 + 9 files changed, 362 insertions(+), 12 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/dialer_test.go diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 89e8e37a6..6c38d0e46 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -15,12 +15,18 @@ package controlapi import ( + "crypto/tls" + "crypto/x509" + "encoding/asn1" "errors" "fmt" + "os" + + "github.com/agent-substrate/substrate/internal/credbundle" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -28,19 +34,38 @@ import ( var ErrWorkerPodNotFound = errors.New("worker pod not found") +// oidPodUID represents the Pod UID within a certificate extension. +var oidPodUID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 6, 1, 3} + +type customExtensionVal struct { + S string `asn1:"utf8"` +} + // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { workerIndexer cache.Indexer ateletIndexer cache.Indexer ateletConns *lru.Cache + // dialCredentials builds the transport credentials used to dial a given + // atelet, keyed on the atelet's expected pod UID. Production wires this to + // per-atelet mTLS; tests can override it with insecure credentials. + dialCredentials func(expectedPodUID string) (credentials.TransportCredentials, error) } -// NewAteletDialer creates a new AteletDialer. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer) *AteletDialer { +// NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath +// are used to build the per-atelet mTLS credentials used for every atelet connection. +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string) *AteletDialer { return &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: lru.New(1024), + dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) { + tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID) + if err != nil { + return nil, err + } + return credentials.NewTLS(tlsConfig), nil + }, } } @@ -73,7 +98,7 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( } selectedAtelet := matchingAtelets[0].(*corev1.Pod) - ateletKey := selectedAtelet.ObjectMeta.Namespace + "/" + selectedAtelet.ObjectMeta.Name + ateletKey := string(selectedAtelet.ObjectMeta.UID) ateletConnAny, ok := d.ateletConns.Get(ateletKey) if ok { @@ -81,12 +106,17 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( } if len(selectedAtelet.Status.PodIPs) == 0 { - return nil, fmt.Errorf("selected atelet %q has no assigned IPs: %w", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name, err) + return nil, fmt.Errorf("selected atelet %q has no assigned IPs", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name) + } + + creds, err := d.dialCredentials(string(selectedAtelet.ObjectMeta.UID)) + if err != nil { + return nil, fmt.Errorf("while building atelet credentials: %w", err) } ateletConn, err := grpc.NewClient( selectedAtelet.Status.PodIPs[0].IP+":8085", - grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithTransportCredentials(creds), grpc.WithStatsHandler(otelgrpc.NewClientHandler()), ) if err != nil { @@ -97,3 +127,71 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( return ateletConn, nil } + +func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { + roots, err := caPoolFromFile(serverCAPath) + if err != nil { + return nil, err + } + + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS12, + GetClientCertificate: credbundle.ClientLoader(clientBundlePath), + // Skip the default verification because the peer is dialed by IP and its + // certificate has no DNS/IP SAN. + InsecureSkipVerify: true, + RootCAs: roots, + VerifyConnection: verifyAteletServerCert(roots, expectedPodUID), + } + + return &tlsConfig, nil +} + +func verifyAteletServerCert(roots *x509.CertPool, expectedPodUID string) func(tls.ConnectionState) error { + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + return fmt.Errorf("server presented no certificate") + } + leaf := cs.PeerCertificates[0] + intermediates := x509.NewCertPool() + for _, c := range cs.PeerCertificates[1:] { + intermediates.AddCert(c) + } + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + return fmt.Errorf("verifying server certificate chain: %w", err) + } + + var foundPodUID customExtensionVal + for _, ext := range leaf.Extensions { + if ext.Id.Equal(oidPodUID) { + if _, err := asn1.Unmarshal(ext.Value, &foundPodUID); err != nil { + return fmt.Errorf("failed to parse oidPodUID extension: %w", err) + } + } + } + if foundPodUID.S == "" { + return fmt.Errorf("certificate is missing oidPodUID extension") + } + if foundPodUID.S != expectedPodUID { + return fmt.Errorf("pod UID is %q does not match expected %q", foundPodUID.S, expectedPodUID) + } + + return nil + } +} + +func caPoolFromFile(path string) (*x509.CertPool, error) { + caBytes, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read CA bundle from %s: %w", path, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caBytes) { + return nil, fmt.Errorf("failed to parse CA bundle from %s", path) + } + return pool, nil +} diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go new file mode 100644 index 000000000..f12a38bda --- /dev/null +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -0,0 +1,152 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "math/big" + "testing" + "time" +) + +// makeTestCA mints a self-signed CA and returns it along with a pool +// containing it as the sole root. +func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509.CertPool) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating CA key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing CA certificate: %v", err) + } + pool := x509.NewCertPool() + pool.AddCert(cert) + return cert, key, pool +} + +// makeLeafCert mints a server leaf certificate signed by the given CA. If +// podUID is non-empty it is embedded as the oidPodUID extension. +func makeLeafCert(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, podUID string) *x509.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating leaf key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if podUID != "" { + podUIDExt, err := asn1.Marshal(customExtensionVal{S: podUID}) + if err != nil { + t.Fatalf("marshaling pod UID extension: %v", err) + } + template.ExtraExtensions = []pkix.Extension{ + {Id: oidPodUID, Critical: false, Value: podUIDExt}, + } + } + der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey) + if err != nil { + t.Fatalf("creating leaf certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing leaf certificate: %v", err) + } + return cert +} + +func TestVerifyAteletServerCert(t *testing.T) { + ca, caKey, roots := makeTestCA(t) + otherCA, otherCAKey, _ := makeTestCA(t) + + const uid = "5a2e1c9f-0b57-4a52-9f6e-2f6d3a1b8c4d" + + tests := []struct { + name string + leaf *x509.Certificate + expectedUID string + wantErr bool + }{ + { + name: "matching UID succeeds", + leaf: makeLeafCert(t, ca, caKey, uid), + expectedUID: uid, + }, + { + name: "mismatched UID fails", + leaf: makeLeafCert(t, ca, caKey, "some-other-uid"), + expectedUID: uid, + wantErr: true, + }, + { + name: "missing pod UID extension fails", + leaf: makeLeafCert(t, ca, caKey, ""), + expectedUID: uid, + wantErr: true, + }, + { + name: "cert from untrusted CA fails", + leaf: makeLeafCert(t, otherCA, otherCAKey, uid), + expectedUID: uid, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + verify := verifyAteletServerCert(roots, tc.expectedUID) + err := verify(tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{tc.leaf}, + }) + if gotErr := err != nil; gotErr != tc.wantErr { + t.Fatalf("verify returned error %v, wantErr=%v", err, tc.wantErr) + } + }) + } + + t.Run("no peer certificate fails", func(t *testing.T) { + verify := verifyAteletServerCert(roots, uid) + if err := verify(tls.ConnectionState{}); err == nil { + t.Fatal("verify succeeded, want error") + } + }) +} diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index 57c08a9c5..8ab5130f0 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -42,6 +42,7 @@ import ( "github.com/redis/go-redis/v9" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -309,7 +310,13 @@ func setupTest(t *testing.T, ns string) *testContext { t.Fatalf("failed to start worker cache: %v", err) } - dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer()) + dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "") + // Dial the fake atelet over insecure transport instead of per-atelet mTLS, + // so DialForWorker's real lookup/dial/cache path is exercised under test. + dialer.dialCredentials = func(_ string) (credentials.TransportCredentials, error) { + return insecure.NewCredentials(), nil + } + service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient) // 5. Start REAL gRPC Server for ATE API diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 4bd8faf07..b8d87adec 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -70,6 +70,9 @@ var ( sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") workerpoolCACerts = pflag.String("workerpool-ca-certs", "", "The file that contains the CA for verifying workerpool client certificates.") + ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle presented as the client certificate when dialing atelet.") + ateletCACerts = pflag.String("atelet-ca-certs", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify atelet serving certificates.") + showVersion = pflag.Bool("version", false, "Print version and exit.") authMode = pflag.String("auth-mode", "mtls", "Auth mode for incoming gRPC: mtls|jwt. 'mtls' (default) relies on transport-level mTLS for client identity. 'jwt' additionally requires a Kubernetes ServiceAccount Bearer token on every RPC. Substrate will drop support for JWT auth mode once the Pod Certificates feature is enabled by default in the minimum supported Kubernetes version.") clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.") @@ -150,8 +153,8 @@ func main() { ateletPodInformerFactory.WaitForCacheSync(stopCh) ateFactory.WaitForCacheSync(stopCh) - dialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer()) - sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, clientset) + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *ateletCACerts) + sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, ateletDialer, clientset) jwtIssuerDiscoveryClient := buildK8sServiceAccountIssuerDiscoveryClient(ctx, *clientJWTCAFile, *clientJWTIssuer) if authModeParsed == ateapiauth.ModeJWT && jwtIssuerDiscoveryClient == nil { @@ -241,6 +244,8 @@ func logFlagValues(ctx context.Context) { slog.String("session-id-jwt-pool", *sessionIDJWTPoolFile), slog.String("session-id-ca-pool", *sessionIDCAPoolFile), slog.String("workerpool-ca-certs", *workerpoolCACerts), + slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), + slog.String("atelet-ca-certs", *ateletCACerts), slog.String("auth-mode", *authMode), ) } diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 9a5eaefaa..9a337f8c4 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -16,6 +16,8 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -34,6 +36,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateerrors" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -53,6 +56,7 @@ import ( "google.golang.org/api/option" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/reflection" "google.golang.org/grpc/status" @@ -65,6 +69,9 @@ var ( port = pflag.Int("port", 8085, "The port to listen on") metricsListenAddr = pflag.String("metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") + grpcServerCredBundle = pflag.String("grpc-server-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle atelet presents as its gRPC serving certificate.") + clientCACerts = pflag.String("client-ca-certs", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify gRPC client certificates.") + gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -173,7 +180,16 @@ func main() { serverboot.Fatal(ctx, "Failed to listen", err) } - svr := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler()), grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor)) + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) + } + + svr := grpc.NewServer( + grpc.Creds(credentials.NewTLS(tlsCfg)), + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), + ) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) @@ -1060,3 +1076,23 @@ func resetActorDirs(atespace, actorName string) error { return nil } + +// ateletServerTLSConfig builds a *tls.Config for a gRPC server that presents the +// credential bundle at servingBundlePath, requires a client certificate +// chaining to a CA in clientCAPath. +func ateletServerTLSConfig(servingBundlePath, clientCAPath string) (*tls.Config, error) { + caBytes, err := os.ReadFile(clientCAPath) + if err != nil { + return nil, fmt.Errorf("read CA bundle %s: %w", clientCAPath, err) + } + clientCAs := x509.NewCertPool() + if !clientCAs.AppendCertsFromPEM(caBytes) { + return nil, fmt.Errorf("parse CA bundle from %s", clientCAPath) + } + return &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: credbundle.Loader(servingBundlePath), + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, + }, nil +} diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go index 6d23da194..379d47fcd 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go @@ -19,6 +19,8 @@ import ( "context" "crypto/rand" "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" "encoding/pem" "fmt" "net/url" @@ -38,6 +40,9 @@ import ( const Name = "podidentity.podcert.ate.dev/identity" const CTBPrefix = "podidentity.podcert.ate.dev:identity:" +// oidPodUID represents the Pod UID within a certificate extension. +var oidPodUID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 6, 1, 3} + type Impl struct { kc kubernetes.Interface caPool *localca.Pool @@ -59,6 +64,10 @@ func (h *Impl) SignerName() string { return Name } +type customExtensionVal struct { + S string `asn1:"utf8"` +} + func (h *Impl) DesiredClusterTrustBundles() []*certsv1beta1.ClusterTrustBundle { name := CTBPrefix + "primary-bundle" @@ -122,18 +131,30 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq } parent := h.caPool.CAs[0].RootCertificate + + podUIDExt, err := asn1.Marshal(customExtensionVal{S: string(pod.UID)}) + if err != nil { + return fmt.Errorf("failed to marshal pod UID %s: %w", pod.UID, err) + } template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, NotAfter: notAfter, URIs: []*url.URL{spiffeURI}, KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, // Link the leaf to its issuing CA by key id so verifiers can disambiguate // a multi-CA trust bundle (e.g. valkey trusts both the servicedns and // podidentity CAs). // https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.1 AuthorityKeyId: parent.SubjectKeyId, + ExtraExtensions: []pkix.Extension{ + { + Id: oidPodUID, + Critical: false, + Value: podUIDExt, + }, + }, } subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index eec53340b..6b54f17c1 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -120,7 +120,9 @@ spec: # servicedns: the apiserver's own gRPC serving cert (DNS SAN api.ate-system.svc). - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" - # podidentity: the apiserver's client identity (SPIFFE) when it dials valkey. + # podidentity: the apiserver's client identity (SPIFFE) when it dials + # valkey and atelet, plus the trust bundle for verifying atelet's + # serving certificate. - name: "podidentity" mountPath: "/run/podidentity.podcert.ate.dev" - name: "session-id-jwt-pool" @@ -160,6 +162,12 @@ spec: signerName: podidentity.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem - name: "session-id-jwt-pool" projected: sources: diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index c9564c40d..86f584f96 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -69,6 +69,8 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atelet args: - --gcp-auth-for-image-pulls=true + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem securityContext: privileged: true env: @@ -105,8 +107,27 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev volumes: - name: run-ateom hostPath: path: /var/lib/ateom-gvisor type: DirectoryOrCreate + # Identity for mutual TLS with the ate-apiserver. atelet is not behind a + # Service, so it uses a podidentity (SPIFFE) cert rather than a servicedns + # serving cert (which requires DNS SANs it would not have); the + # clusterTrustBundle verifies the apiserver's client certificate. + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 8463bfd79..6b28ff2fd 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -33,6 +33,8 @@ patches: args: - --gcp-auth-for-image-pulls=false - --localhost-registry-replacement=kind-registry:5000 + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: http://opentelemetry-collector.otel-system.svc:4317 From 3edfcf48cdbda7bfc9106ce44161c386274dc208 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Fri, 10 Jul 2026 16:55:09 -0700 Subject: [PATCH 3/9] Present a real client cert from atecontroller to ateapi in mtls mode The --ateapi-client-cert path is provided via the manifest instead of a code default, matching how the other cert-material flags are wired. --- cmd/atecontroller/main.go | 14 +- internal/ateapiauth/client.go | 49 ++++-- internal/ateapiauth/client_test.go | 200 +++++++++++++++++++++- manifests/ate-install/ate-controller.yaml | 25 +++ 4 files changed, 269 insertions(+), 19 deletions(-) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index e96bdb242..48069eab3 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -39,10 +39,11 @@ var ( ateAPIConnSpec = pflag.String("ateapi-conn-spec", "dns:///api.ate-system.svc:443", "") - ateapiAuthMode = pflag.String("ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.") - ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") + ateapiAuthMode = pflag.String("ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) verifies the server cert and presents the pod-projected client certificate. 'jwt' verifies the server cert and sends a Bearer SA token.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") ateapiTokenFile = pflag.String("ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") + ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required for mtls.") ) func init() { @@ -61,10 +62,11 @@ func main() { } dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: mode, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - TokenFile: *ateapiTokenFile, + Mode: mode, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + TokenFile: *ateapiTokenFile, + ClientCredBundle: *ateapiClientCert, }) if err != nil { setupLog.Error(err, "building ateapi dial options") diff --git a/internal/ateapiauth/client.go b/internal/ateapiauth/client.go index 8ae0807fe..c88a16e22 100644 --- a/internal/ateapiauth/client.go +++ b/internal/ateapiauth/client.go @@ -22,6 +22,7 @@ import ( "os" "strings" + "github.com/agent-substrate/substrate/internal/credbundle" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) @@ -33,17 +34,16 @@ const ( // ClientConfig configures how to dial the ateapi gRPC server. // -// - Mode=ModeMTLS: insecure TLS dial (InsecureSkipVerify=true). Client -// identity is expected to come from mTLS credentials projected into -// the pod (servicedns.podcert.ate.dev). No app-level credentials. +// - Mode=ModeMTLS: mutual TLS. Validates the server cert against CAFile and +// presents the client certificate from ClientCredBundle, re-read on every +// handshake so in-place pod-certificate rotations are picked up. // - Mode=ModeJWT: validates the server cert against CAFile, sends a Bearer // token from TokenFile as per-RPC credentials. type ClientConfig struct { Mode Mode // CAFile is a PEM file containing CA certs that sign the server cert. - // Required in all modes. Ignored for ModeMTLS until mTLS verification is - // fully wired. + // Required in all modes. CAFile string // ServerName overrides SNI / hostname verification. Optional. @@ -52,6 +52,11 @@ type ClientConfig struct { // TokenFile is a path to a Kubernetes projected ServiceAccount token used // as a Bearer credential. Required for ModeJWT. TokenFile string + + // ClientCredBundle is a PEM file containing the client certificate chain + // and PKCS8 private key presented to the server. Required for ModeMTLS + // and ignored for ModeJWT. + ClientCredBundle string } // DialOptions returns the grpc.DialOption set described by cfg, suitable to @@ -62,7 +67,19 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { } switch cfg.Mode { case "", ModeMTLS: - tlsCfg := &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit opt-in + if cfg.ClientCredBundle == "" { + return nil, fmt.Errorf("ateapiauth: mtls mode requires ClientCredBundle") + } + pool, err := loadCAPool(cfg.CAFile) + if err != nil { + return nil, err + } + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: pool, + ServerName: cfg.ServerName, + GetClientCertificate: credbundle.ClientLoader(cfg.ClientCredBundle), + } return []grpc.DialOption{ grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), }, nil @@ -71,13 +88,9 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { if cfg.TokenFile == "" { return nil, fmt.Errorf("ateapiauth: jwt mode requires TokenFile") } - caPEM, err := os.ReadFile(cfg.CAFile) + pool, err := loadCAPool(cfg.CAFile) if err != nil { - return nil, fmt.Errorf("ateapiauth: reading CA file: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("ateapiauth: no certificates found in CA file %q", cfg.CAFile) + return nil, err } tlsCfg := &tls.Config{ MinVersion: tls.VersionTLS12, @@ -94,6 +107,18 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { } } +func loadCAPool(caFile string) (*x509.CertPool, error) { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("ateapiauth: reading CA file: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("ateapiauth: no certificates found in CA file %q", caFile) + } + return pool, nil +} + // fileTokenCreds reads a Kubernetes projected SA token from disk for every // RPC. Kubernetes refreshes the file in place; reading it each time picks up // rotations. diff --git a/internal/ateapiauth/client_test.go b/internal/ateapiauth/client_test.go index d8f271ee3..512df7f42 100644 --- a/internal/ateapiauth/client_test.go +++ b/internal/ateapiauth/client_test.go @@ -14,7 +14,29 @@ package ateapiauth -import "testing" +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) func TestDialOptionsRequiresCAFile(t *testing.T) { for _, mode := range []Mode{ModeMTLS, ModeJWT} { @@ -29,3 +51,179 @@ func TestDialOptionsRequiresCAFile(t *testing.T) { }) } } + +func TestDialOptionsMTLSRequiresClientCredBundle(t *testing.T) { + _, err := DialOptions(ClientConfig{ + Mode: ModeMTLS, + CAFile: "ca.pem", + }) + if err == nil { + t.Fatalf("DialOptions() error = nil, want error for missing ClientCredBundle") + } +} + +// TestDialOptionsMTLSHandshake dials a server that requires and verifies +// client certificates — the configuration ateapi will move to — and checks +// that DialOptions in mtls mode completes the handshake, while a +// certificate-less client is rejected. +func TestDialOptionsMTLSHandshake(t *testing.T) { + ca := newTestCA(t) + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.pem") + writeFile(t, caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.certDER})) + + clientBundle := filepath.Join(dir, "client-bundle.pem") + writeFile(t, clientBundle, ca.issueClientBundle(t, "spiffe://cluster.local/ns/ate-system/sa/ate-controller")) + + serverCert := ca.issueServerCert(t) + caPool := x509.NewCertPool() + caPool.AddCert(ca.cert) + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: caPool, + MinVersion: tls.VersionTLS12, + }))) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + go srv.Serve(lis) + defer srv.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + t.Run("with client cert", func(t *testing.T) { + opts, err := DialOptions(ClientConfig{ + Mode: ModeMTLS, + CAFile: caFile, + ClientCredBundle: clientBundle, + }) + if err != nil { + t.Fatalf("DialOptions() error = %v", err) + } + // Unimplemented proves the TLS handshake and HTTP/2 setup succeeded + // and the RPC reached the server (which has no services registered). + if code := invokeCode(ctx, t, lis.Addr().String(), opts); code != codes.Unimplemented { + t.Fatalf("Invoke() code = %v, want %v", code, codes.Unimplemented) + } + }) + + t.Run("without client cert is rejected", func(t *testing.T) { + opts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + RootCAs: caPool, + MinVersion: tls.VersionTLS12, + }))} + if code := invokeCode(ctx, t, lis.Addr().String(), opts); code == codes.Unimplemented { + t.Fatalf("Invoke() code = %v, want handshake failure", code) + } + }) +} + +func invokeCode(ctx context.Context, t *testing.T, target string, opts []grpc.DialOption) codes.Code { + t.Helper() + conn, err := grpc.NewClient(target, opts...) + if err != nil { + t.Fatalf("grpc.NewClient() error = %v", err) + } + defer conn.Close() + err = conn.Invoke(ctx, "/test.NoSuchService/Ping", &emptypb.Empty{}, &emptypb.Empty{}) + return status.Code(err) +} + +type testCA struct { + cert *x509.Certificate + certDER []byte + key *ecdsa.PrivateKey +} + +func newTestCA(t *testing.T) *testCA { + t.Helper() + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("create CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse CA certificate: %v", err) + } + return &testCA{cert: cert, certDER: der, key: key} +} + +// issueClientBundle returns a PEM credential bundle (leaf certificate + PKCS8 +// private key) for a client certificate carrying the given SPIFFE URI SAN. +func (ca *testCA) issueClientBundle(t *testing.T, spiffeID string) []byte { + t.Helper() + uri, err := url.Parse(spiffeID) + if err != nil { + t.Fatalf("parse SPIFFE ID: %v", err) + } + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "test-client"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + URIs: []*url.URL{uri}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("create client certificate: %v", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal PKCS8 key: %v", err) + } + return append( + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})..., + ) +} + +func (ca *testCA) issueServerCert(t *testing.T) tls.Certificate { + t.Helper() + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(3), + Subject: pkix.Name{CommonName: "test-server"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("create server certificate: %v", err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +func generateKey(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + return key +} + +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 65ee46075..a6955bb81 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -90,6 +90,7 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atecontroller args: - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem ports: - name: metrics containerPort: 8080 @@ -97,3 +98,27 @@ spec: - name: healthz containerPort: 8081 protocol: TCP + volumeMounts: + # Controller's own client identity presented to ateapi. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: "servicedns-ca" + mountPath: "/run/servicedns-ca" + volumes: + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem From ab2e515845a1f272bb6eebcc9598988010a89956 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Sat, 11 Jul 2026 22:07:12 -0700 Subject: [PATCH 4/9] Unify atenet router's ateapi auth flags Drop --ateapi-ca-certs: it duplicated --ateapi-ca-file (both are the CA bundle used to verify ateapi's serving cert, split by auth mode). The router now dials ateapi via ateapiauth.DialOptions in both modes, like atecontroller, instead of hand-rolling the mtls TLS config. This also gains TLS 1.2 minimum and --ateapi-server-name support in mtls mode. Path defaults move to the manifest (--ateapi-ca-file previously defaulted to the SA ca.crt, which never matches ateapi's servicedns- signed serving cert), and the router's servicedns-ca trust bundle now mounts at /run/servicedns-ca, matching ate-controller and deduping the jwt overlay's second mount of the same bundle. --- cmd/atenet/internal/router.go | 7 +- cmd/atenet/internal/router/router.go | 98 +++++------------------ cmd/atenet/internal/router/status_test.go | 2 +- manifests/ate-install/atenet-router.yaml | 9 ++- 4 files changed, 27 insertions(+), 89 deletions(-) diff --git a/cmd/atenet/internal/router.go b/cmd/atenet/internal/router.go index d62962a44..e3f0a08b8 100644 --- a/cmd/atenet/internal/router.go +++ b/cmd/atenet/internal/router.go @@ -58,10 +58,9 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)") cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)") - cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) presents the podidentity client cert (--ateapi-client-cert) and verifies ateapi's serving cert against the servicedns trust bundle (--ateapi-ca-certs). 'jwt' verifies the server cert and sends a Bearer SA token.") - cmd.Flags().StringVar(&cfg.AteapiClientCertPath, "ateapi-client-cert", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Path to the podidentity credential bundle the router presents as its client cert to ateapi (mtls mode).") - cmd.Flags().StringVar(&cfg.AteapiCACertsPath, "ateapi-ca-certs", "/run/servicedns-ca.podcert.ate.dev/trust-bundle.pem", "Path to the servicedns trust bundle used to verify ateapi's serving cert (mtls mode).") - cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") + cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) verifies the server cert (--ateapi-ca-file) and presents the pod-projected client certificate (--ateapi-client-cert). 'jwt' verifies the server cert and sends a Bearer SA token.") + cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", "", "PEM file with CAs trusted to verify the ateapi server cert. Required.") + cmd.Flags().StringVar(&cfg.AteapiClientCertPath, "ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required for mtls.") cmd.Flags().StringVar(&cfg.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") cmd.Flags().StringVar(&cfg.AteapiTokenFile, "ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 7f18045b8..b6b934852 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -18,7 +18,6 @@ import ( "context" "crypto/rand" "crypto/rsa" - "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -40,7 +39,6 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" @@ -50,7 +48,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/agent-substrate/substrate/internal/ateapiauth" - "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/serverboot" v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -87,17 +84,16 @@ type RouterConfig struct { // Envoy reports tracing spans to. Empty disables Envoy-side tracing. OtlpCollectorAddress string - AteapiAuthMode string - AteapiCAFile string - AteapiServerName string - AteapiTokenFile string - - // mTLS mode (ateapi-auth=mtls) material: the router presents - // AteapiClientCertPath (the podidentity credential bundle) as its client - // cert and verifies ateapi's serving cert against AteapiCACertsPath (the - // servicedns trust bundle). + // Client auth to ateapi: AteapiCAFile verifies ateapi's serving cert in + // both modes (the servicedns trust bundle in-cluster); in mtls mode the + // router additionally presents AteapiClientCertPath (the podidentity + // credential bundle) as its client cert, in jwt mode it sends a Bearer + // token from AteapiTokenFile instead. + AteapiAuthMode string + AteapiCAFile string AteapiClientCertPath string - AteapiCACertsPath string + AteapiServerName string + AteapiTokenFile string } // RouterServer instantiates and coordinates runtime threads executing system modules. @@ -159,53 +155,6 @@ func NewRouterServer(cfg RouterConfig) (*RouterServer, error) { }, nil } -// ateapiTransportCreds builds the mTLS credentials the router uses to dial -// ateapi in ateapi-auth=mtls mode. When both the servicedns trust bundle and -// the podidentity client credential bundle are present (the in-cluster case, -// mounted via projected pod-certificate volumes), it verifies ateapi's serving -// cert against the servicedns trust bundle and presents its own podidentity -// SPIFFE client cert. When that material is absent, it returns an error rather -// than falling back to an insecure connection. -func ateapiTransportCreds(cfg RouterConfig) (credentials.TransportCredentials, error) { - tlsCfg, err := ateapiTLSConfig(cfg) - if err != nil { - return nil, err - } - return credentials.NewTLS(tlsCfg), nil -} - -func ateapiTLSConfig(cfg RouterConfig) (*tls.Config, error) { - haveCA := fileExists(cfg.AteapiCACertsPath) - haveClientCert := fileExists(cfg.AteapiClientCertPath) - - if !haveCA || !haveClientCert { - return nil, fmt.Errorf("ateapi mTLS material not found: ca-certs=%q client-cert=%q", - cfg.AteapiCACertsPath, cfg.AteapiClientCertPath) - } - - caBytes, err := os.ReadFile(cfg.AteapiCACertsPath) - if err != nil { - return nil, fmt.Errorf("read ateapi CA certs: %w", err) - } - rootCAs := x509.NewCertPool() - if !rootCAs.AppendCertsFromPEM(caBytes) { - return nil, fmt.Errorf("parse ateapi CA certs from %s", cfg.AteapiCACertsPath) - } - - return &tls.Config{ - RootCAs: rootCAs, - GetClientCertificate: credbundle.ClientLoader(cfg.AteapiClientCertPath), - }, nil -} - -func fileExists(path string) bool { - if path == "" { - return false - } - _, err := os.Stat(path) - return err == nil -} - func (s *RouterServer) Run(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -254,26 +203,15 @@ func (s *RouterServer) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("invalid --ateapi-auth: %w", err) } - var dialOpts []grpc.DialOption - switch authMode { - case ateapiauth.ModeMTLS: - // Real mutual TLS: present the router's podidentity client cert and - // verify ateapi's serving cert against the servicedns trust bundle. - creds, err := ateapiTransportCreds(s.cfg) - if err != nil { - return fmt.Errorf("building ateapi mTLS credentials: %w", err) - } - dialOpts = []grpc.DialOption{grpc.WithTransportCredentials(creds)} - default: - dialOpts, err = ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: authMode, - CAFile: s.cfg.AteapiCAFile, - ServerName: s.cfg.AteapiServerName, - TokenFile: s.cfg.AteapiTokenFile, - }) - if err != nil { - return fmt.Errorf("building ateapi dial options: %w", err) - } + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + Mode: authMode, + CAFile: s.cfg.AteapiCAFile, + ServerName: s.cfg.AteapiServerName, + TokenFile: s.cfg.AteapiTokenFile, + ClientCredBundle: s.cfg.AteapiClientCertPath, + }) + if err != nil { + return fmt.Errorf("building ateapi dial options: %w", err) } dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) conn, err := grpc.NewClient( diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index 64d003ca8..b1fa5924a 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -67,7 +67,7 @@ func TestStatuszEndpoint(t *testing.T) { ExtprocPort: 50051, TemplatesFile: tmpFile.Name(), MetricsAddr: "127.0.0.1:0", - AteapiCACertsPath: caPath, + AteapiCAFile: caPath, AteapiClientCertPath: clientCertPath, } diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index dfc109087..ad6f9dfd6 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -131,14 +131,15 @@ spec: - "--port-xds=18000" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - - "--ateapi-address=api.ate-system.svc:443" - - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - - "--ateapi-ca-certs=/run/servicedns-ca.podcert.ate.dev/trust-bundle.pem" - "--status-port=4040" - "--port-https=8443" - "--envoy-cert-path=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--otlp-collector-address=opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317" + # Client auth to ateapi (mtls): verify the serving cert against the + # servicedns trust bundle and present the podidentity client cert. + - "--ateapi-address=api.ate-system.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" env: - name: POD_NAME valueFrom: @@ -171,7 +172,7 @@ spec: mountPath: "/run/podidentity.podcert.ate.dev" # Trust bundle used to verify ateapi's servicedns serving cert. - name: "servicedns-ca" - mountPath: "/run/servicedns-ca.podcert.ate.dev" + mountPath: "/run/servicedns-ca" - name: envoy image: envoyproxy/envoy:v1.30-latest command: From 486fe992e313c50b76eaf20790926413954bb37b Mon Sep 17 00:00:00 2001 From: zoezhao Date: Sat, 11 Jul 2026 22:17:53 -0700 Subject: [PATCH 5/9] Move atenet router cmd/config into the router package Port the file-layout refactors from PR #237 that were lost when the mtls branch was redone on a newer main: - Move NewRouterCmd from cmd/atenet/internal into the router package as cmd.go; root.go now calls router.NewRouterCmd(). - Move the config struct to config.go, unexport it (routerConfig), and group the ateapi client-auth fields under a nested authConfig. - Drop the self-signed Envoy cert fallback: --envoy-cert-path is now required for HTTPS serving; SetTlsConfig takes just (port, path) and xDS always references the cert by filename. --- cmd/ateapi/internal/controlapi/dialer.go | 3 - cmd/ateapi/main.go | 2 +- cmd/atenet/internal/root.go | 3 +- .../internal/{router.go => router/cmd.go} | 19 ++-- cmd/atenet/internal/router/config.go | 57 ++++++++++ cmd/atenet/internal/router/controller.go | 4 +- cmd/atenet/internal/router/envoyrunner.go | 4 +- cmd/atenet/internal/router/health.go | 4 +- cmd/atenet/internal/router/router.go | 103 ++---------------- cmd/atenet/internal/router/status_test.go | 24 ++-- cmd/atenet/internal/router/xds.go | 33 ++---- cmd/atenet/internal/router/xds_test.go | 2 +- 12 files changed, 104 insertions(+), 154 deletions(-) rename cmd/atenet/internal/{router.go => router/cmd.go} (73%) create mode 100644 cmd/atenet/internal/router/config.go diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 6c38d0e46..4d856bce9 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -173,9 +173,6 @@ func verifyAteletServerCert(roots *x509.CertPool, expectedPodUID string) func(tl } } } - if foundPodUID.S == "" { - return fmt.Errorf("certificate is missing oidPodUID extension") - } if foundPodUID.S != expectedPodUID { return fmt.Errorf("pod UID is %q does not match expected %q", foundPodUID.S, expectedPodUID) } diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index b8d87adec..1961ce493 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -244,9 +244,9 @@ func logFlagValues(ctx context.Context) { slog.String("session-id-jwt-pool", *sessionIDJWTPoolFile), slog.String("session-id-ca-pool", *sessionIDCAPoolFile), slog.String("workerpool-ca-certs", *workerpoolCACerts), + slog.String("auth-mode", *authMode), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), slog.String("atelet-ca-certs", *ateletCACerts), - slog.String("auth-mode", *authMode), ) } diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index 7c7e66806..53ae85d13 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -18,6 +18,7 @@ import ( "fmt" "os" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router" "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/cobra" ) @@ -37,6 +38,6 @@ func Execute() { } func init() { - rootCmd.AddCommand(NewRouterCmd()) + rootCmd.AddCommand(router.NewRouterCmd()) rootCmd.AddCommand(NewDnsCmd()) } diff --git a/cmd/atenet/internal/router.go b/cmd/atenet/internal/router/cmd.go similarity index 73% rename from cmd/atenet/internal/router.go rename to cmd/atenet/internal/router/cmd.go index e3f0a08b8..873731929 100644 --- a/cmd/atenet/internal/router.go +++ b/cmd/atenet/internal/router/cmd.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package router import ( "fmt" @@ -20,18 +20,17 @@ import ( "github.com/spf13/cobra" - "github.com/agent-substrate/substrate/cmd/atenet/internal/router" "github.com/agent-substrate/substrate/internal/ateapiauth" ) func NewRouterCmd() *cobra.Command { - var cfg router.RouterConfig + var cfg routerConfig cmd := &cobra.Command{ Use: "router", Short: "Router components including xDS server and Envoy ExtProc gateway processing server", RunE: func(cmd *cobra.Command, args []string) error { - srv, err := router.NewRouterServer(cfg) + srv, err := NewRouterServer(cfg) if err != nil { return fmt.Errorf("failed to create router server: %w", err) } @@ -56,13 +55,13 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)") cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services") cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") - cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)") + cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)") - cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) verifies the server cert (--ateapi-ca-file) and presents the pod-projected client certificate (--ateapi-client-cert). 'jwt' verifies the server cert and sends a Bearer SA token.") - cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", "", "PEM file with CAs trusted to verify the ateapi server cert. Required.") - cmd.Flags().StringVar(&cfg.AteapiClientCertPath, "ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required for mtls.") - cmd.Flags().StringVar(&cfg.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") - cmd.Flags().StringVar(&cfg.AteapiTokenFile, "ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") + cmd.Flags().StringVar(&cfg.Auth.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) verifies the server cert (--ateapi-ca-file) and presents the pod-projected client certificate (--ateapi-client-cert). 'jwt' verifies the server cert and sends a Bearer SA token.") + cmd.Flags().StringVar(&cfg.Auth.AteapiCAFile, "ateapi-ca-file", "", "PEM file with CAs trusted to verify the ateapi server cert. Required.") + cmd.Flags().StringVar(&cfg.Auth.AteapiClientCertPath, "ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required for mtls.") + cmd.Flags().StringVar(&cfg.Auth.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") + cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") return cmd } diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go new file mode 100644 index 000000000..2b7d3732b --- /dev/null +++ b/cmd/atenet/internal/router/config.go @@ -0,0 +1,57 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "time" +) + +// authConfig holds the router's client-auth settings for dialing ateapi. +// AteapiCAFile verifies ateapi's serving cert in both modes (the servicedns +// trust bundle in-cluster); in mtls mode the router additionally presents +// AteapiClientCertPath (the podidentity credential bundle) as its client +// cert, in jwt mode it sends a Bearer token from AteapiTokenFile instead. +type authConfig struct { + AteapiAuthMode string + AteapiCAFile string + AteapiClientCertPath string + AteapiServerName string + AteapiTokenFile string +} + +// routerConfig holds deployment setup and endpoint options for the router node instance. +type routerConfig struct { + Standalone bool + Namespace string + Kubeconfig string + AteapiAddr string + HttpPort int + XdsPort int + ExtprocPort int + ExtprocAddr string + EnvoyImage string + TemplatesFile string + StatusPort int + HealthInterval time.Duration + HttpsPort int + EnvoyCertPath string + LogLevel string + MetricsAddr string + // OtlpCollectorAddress is the host:port of the OTLP gRPC collector that + // Envoy reports tracing spans to. Empty disables Envoy-side tracing. + OtlpCollectorAddress string + + Auth authConfig +} diff --git a/cmd/atenet/internal/router/controller.go b/cmd/atenet/internal/router/controller.go index 1b0a1ed18..e66243d5b 100644 --- a/cmd/atenet/internal/router/controller.go +++ b/cmd/atenet/internal/router/controller.go @@ -28,7 +28,7 @@ import ( type Controller struct { k8sClient client.Client clientset kubernetes.Interface - cfg RouterConfig + cfg routerConfig xdsSrv *XdsServer extprocSrv *ExtProcServer @@ -39,7 +39,7 @@ type Controller struct { func NewController( k8sClient client.Client, clientset kubernetes.Interface, - cfg RouterConfig, + cfg routerConfig, xdsSrv *XdsServer, extprocSrv *ExtProcServer, ) *Controller { diff --git a/cmd/atenet/internal/router/envoyrunner.go b/cmd/atenet/internal/router/envoyrunner.go index df8765c62..4eee29303 100644 --- a/cmd/atenet/internal/router/envoyrunner.go +++ b/cmd/atenet/internal/router/envoyrunner.go @@ -37,10 +37,10 @@ const ( // Envoy proxy instance running inside Kubernetes. type envoyrunner struct { k8sClient client.Client - cfg RouterConfig + cfg routerConfig } -func newEnvoyRunner(k8sClient client.Client, cfg RouterConfig) *envoyrunner { +func newEnvoyRunner(k8sClient client.Client, cfg routerConfig) *envoyrunner { return &envoyrunner{ k8sClient: k8sClient, cfg: cfg, diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index d9e1fddab..022b47ab1 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -54,11 +54,11 @@ type routerHealth struct { interval time.Duration clientset kubernetes.Interface apiClient ateapipb.ControlClient - cfg RouterConfig + cfg routerConfig envoyClient *http.Client } -func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg RouterConfig) *routerHealth { +func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg routerConfig) *routerHealth { if interval <= 0 { interval = time.Second } diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index b6b934852..b28372a81 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -16,22 +16,15 @@ package router import ( "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "errors" "fmt" "log/slog" - "math/big" "net" "net/http" "os" "os/signal" "strings" "syscall" - "time" "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -62,43 +55,9 @@ func init() { utilruntime.Must(v1alpha1.AddToScheme(scheme)) } -// RouterConfig holds deployment setup and endpoint options for the router node instance. -type RouterConfig struct { - Standalone bool - Namespace string - Kubeconfig string - AteapiAddr string - HttpPort int - XdsPort int - ExtprocPort int - ExtprocAddr string - EnvoyImage string - TemplatesFile string - StatusPort int - HealthInterval time.Duration - HttpsPort int - EnvoyCertPath string - LogLevel string - MetricsAddr string - // OtlpCollectorAddress is the host:port of the OTLP gRPC collector that - // Envoy reports tracing spans to. Empty disables Envoy-side tracing. - OtlpCollectorAddress string - - // Client auth to ateapi: AteapiCAFile verifies ateapi's serving cert in - // both modes (the servicedns trust bundle in-cluster); in mtls mode the - // router additionally presents AteapiClientCertPath (the podidentity - // credential bundle) as its client cert, in jwt mode it sends a Bearer - // token from AteapiTokenFile instead. - AteapiAuthMode string - AteapiCAFile string - AteapiClientCertPath string - AteapiServerName string - AteapiTokenFile string -} - // RouterServer instantiates and coordinates runtime threads executing system modules. type RouterServer struct { - cfg RouterConfig + cfg routerConfig Cmd *cobra.Command k8sClient client.Client @@ -109,7 +68,7 @@ type RouterServer struct { atStore atStore } -func NewRouterServer(cfg RouterConfig) (*RouterServer, error) { +func NewRouterServer(cfg routerConfig) (*RouterServer, error) { var k8sClient client.Client var clientset kubernetes.Interface @@ -199,16 +158,16 @@ func (s *RouterServer) Run(ctx context.Context) error { go serverboot.StartMetricsServer(ctx, serverboot.MetricsServerOptions{Addr: s.cfg.MetricsAddr}) - authMode, err := ateapiauth.ParseMode(s.cfg.AteapiAuthMode) + authMode, err := ateapiauth.ParseMode(s.cfg.Auth.AteapiAuthMode) if err != nil { return fmt.Errorf("invalid --ateapi-auth: %w", err) } dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ Mode: authMode, - CAFile: s.cfg.AteapiCAFile, - ServerName: s.cfg.AteapiServerName, - TokenFile: s.cfg.AteapiTokenFile, - ClientCredBundle: s.cfg.AteapiClientCertPath, + CAFile: s.cfg.Auth.AteapiCAFile, + ServerName: s.cfg.Auth.AteapiServerName, + TokenFile: s.cfg.Auth.AteapiTokenFile, + ClientCredBundle: s.cfg.Auth.AteapiClientCertPath, }) if err != nil { return fmt.Errorf("building ateapi dial options: %w", err) @@ -234,17 +193,7 @@ func (s *RouterServer) Run(ctx context.Context) error { return fmt.Errorf("configure OTLP collector: %w", err) } - var certContent, keyContent string - if s.cfg.EnvoyCertPath == "" { - slog.InfoContext(ctx, "No Envoy certificate path provided, generating self-signed certificate for testing") - var err error - certContent, keyContent, err = generateSelfSignedCert() - if err != nil { - return fmt.Errorf("failed to generate self-signed cert: %w", err) - } - } - - xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath, certContent, keyContent) + xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath) if s.extprocSrv == nil { routeDuration, err := newRouteDurationHistogram() if err != nil { @@ -320,39 +269,3 @@ func (s *RouterServer) Run(ctx context.Context) error { return g.Wait() } - -func generateSelfSignedCert() (string, string, error) { - priv, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return "", "", err - } - - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - Organization: []string{"Substrate Local Test"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 365), - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - DNSNames: []string{"localhost"}, - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) - if err != nil { - return "", "", err - } - - certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - - privBytes, err := x509.MarshalPKCS8PrivateKey(priv) - if err != nil { - return "", "", err - } - keyPem := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) - - return string(certPem), string(keyPem), nil -} diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index b1fa5924a..116359ce8 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -58,17 +58,19 @@ func TestStatuszEndpoint(t *testing.T) { // Run() dials ateapi in mtls mode, which requires real TLS material; generate it. caPath, clientCertPath := writeTestTLSMaterial(t) - cfg := RouterConfig{ - Standalone: true, - Namespace: "default", - StatusPort: httpPort, - HttpPort: 8080, - XdsPort: 18000, - ExtprocPort: 50051, - TemplatesFile: tmpFile.Name(), - MetricsAddr: "127.0.0.1:0", - AteapiCAFile: caPath, - AteapiClientCertPath: clientCertPath, + cfg := routerConfig{ + Standalone: true, + Namespace: "default", + StatusPort: httpPort, + HttpPort: 8080, + XdsPort: 18000, + ExtprocPort: 50051, + TemplatesFile: tmpFile.Name(), + MetricsAddr: "127.0.0.1:0", + Auth: authConfig{ + AteapiCAFile: caPath, + AteapiClientCertPath: clientCertPath, + }, } srv, err := NewRouterServer(cfg) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e7e1602a9..3d52657af 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -80,10 +80,8 @@ type XdsServer struct { mu sync.Mutex - httpsPort int - certPath string - certContent string - keyContent string + httpsPort int + certPath string otlpHost string otlpPort uint32 @@ -111,13 +109,11 @@ func (x *XdsServer) SetConfig(ingressPort int, extprocPort int, extprocAddr stri x.extprocAddr = extprocAddr } -func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string, certContent string, keyContent string) { +func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string) { x.mu.Lock() defer x.mu.Unlock() x.httpsPort = httpsPort x.certPath = certPath - x.certContent = certContent - x.keyContent = keyContent } // SetOtlpCollector enables Envoy-side tracing pointed at the OTLP gRPC @@ -576,30 +572,15 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { } func (x *XdsServer) buildTlsCertificate() *tlsv3.TlsCertificate { - if x.certPath != "" { - return &tlsv3.TlsCertificate{ - CertificateChain: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{ - Filename: x.certPath, - }, - }, - PrivateKey: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{ - Filename: x.certPath, // Assuming combined file - }, - }, - } - } - return &tlsv3.TlsCertificate{ CertificateChain: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: x.certContent, + Specifier: &corev3.DataSource_Filename{ + Filename: x.certPath, }, }, PrivateKey: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: x.keyContent, + Specifier: &corev3.DataSource_Filename{ + Filename: x.certPath, // Assuming combined file }, }, } diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 92e347648..9a2c2460f 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -141,7 +141,7 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(8085, 50053, "127.0.0.1") - server.SetTlsConfig(8443, "", "dummy-cert", "dummy-key") + server.SetTlsConfig(8443, "") err := server.UpdateSnapshot() if err != nil { From e39db9fa2f8ffe036a9c4913dc0e4ae3358cb983 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Fri, 17 Jul 2026 09:35:50 -0700 Subject: [PATCH 6/9] Updated TLS version. Limited ExtKeyUsageServerAuth to only Atelet. --- cmd/ateapi/internal/controlapi/dialer.go | 5 +- cmd/ateapi/main.go | 2 +- cmd/atelet/main.go | 2 +- .../podidentitysigner/podidentitysigner.go | 18 +- .../podidentitysigner_test.go | 64 ++++++ internal/ateapiauth/client.go | 4 +- internal/ateapiauth/client_test.go | 22 +-- .../google.golang.org/grpc/health/client.go | 117 +++++++++++ .../google.golang.org/grpc/health/logging.go | 23 +++ .../google.golang.org/grpc/health/producer.go | 106 ++++++++++ .../google.golang.org/grpc/health/server.go | 187 ++++++++++++++++++ vendor/modules.txt | 1 + 12 files changed, 534 insertions(+), 17 deletions(-) create mode 100644 cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go create mode 100644 vendor/google.golang.org/grpc/health/client.go create mode 100644 vendor/google.golang.org/grpc/health/logging.go create mode 100644 vendor/google.golang.org/grpc/health/producer.go create mode 100644 vendor/google.golang.org/grpc/health/server.go diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 4d856bce9..a865cfffa 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -135,7 +135,7 @@ func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls } tlsConfig := tls.Config{ - MinVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS13, GetClientCertificate: credbundle.ClientLoader(clientBundlePath), // Skip the default verification because the peer is dialed by IP and its // certificate has no DNS/IP SAN. @@ -173,6 +173,9 @@ func verifyAteletServerCert(roots *x509.CertPool, expectedPodUID string) func(tl } } } + if foundPodUID.S == "" || expectedPodUID == "" { + return fmt.Errorf("found Pod UID == %q, expected podUID == %q", foundPodUID.S, expectedPodUID) + } if foundPodUID.S != expectedPodUID { return fmt.Errorf("pod UID is %q does not match expected %q", foundPodUID.S, expectedPodUID) } diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 1961ce493..1ad88c38b 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -290,7 +290,7 @@ func connectRedis(ctx context.Context) (*redis.ClusterClient, error) { } func buildRedisTLSConfig(ctx context.Context) (*tls.Config, error) { - tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS13} if *redisCACerts != "" { ca, err := os.ReadFile(*redisCACerts) if err != nil { diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 9a337f8c4..207ccef29 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1090,7 +1090,7 @@ func ateletServerTLSConfig(servingBundlePath, clientCAPath string) (*tls.Config, return nil, fmt.Errorf("parse CA bundle from %s", clientCAPath) } return &tls.Config{ - MinVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS13, GetCertificate: credbundle.Loader(servingBundlePath), ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs, diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go index 379d47fcd..c8fcf00dc 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go @@ -43,6 +43,22 @@ const CTBPrefix = "podidentity.podcert.ate.dev:identity:" // oidPodUID represents the Pod UID within a certificate extension. var oidPodUID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 6, 1, 3} +// atelet's identity, as installed by manifests/ate-install/atelet.yaml. Pods +// running as atelet serve TLS (e.g. to ate-apiserver), so their certs also +// carry the serverAuth EKU. +const ( + ateletNamespace = "ate-system" + ateletServiceAccount = "atelet" +) + +func extKeyUsages(namespace, serviceAccount string) []x509.ExtKeyUsage { + usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + if namespace == ateletNamespace && serviceAccount == ateletServiceAccount { + usages = append(usages, x509.ExtKeyUsageServerAuth) + } + return usages +} + type Impl struct { kc kubernetes.Interface caPool *localca.Pool @@ -142,7 +158,7 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq NotAfter: notAfter, URIs: []*url.URL{spiffeURI}, KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: extKeyUsages(pcr.ObjectMeta.Namespace, pcr.Spec.ServiceAccountName), // Link the leaf to its issuing CA by key id so verifiers can disambiguate // a multi-CA trust bundle (e.g. valkey trusts both the servicedns and // podidentity CAs). diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go new file mode 100644 index 000000000..15300cb3d --- /dev/null +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podidentitysigner + +import ( + "crypto/x509" + "slices" + "testing" +) + +func TestExtKeyUsages(t *testing.T) { + testCases := []struct { + name string + namespace string + serviceAccount string + want []x509.ExtKeyUsage + }{ + { + name: "atelet gets clientAuth and serverAuth", + namespace: "ate-system", + serviceAccount: "atelet", + want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + }, + { + name: "other SA in ate-system is client-only", + namespace: "ate-system", + serviceAccount: "ate-controller", + want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + { + name: "atelet SA name in another namespace is client-only", + namespace: "default", + serviceAccount: "atelet", + want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + { + name: "ordinary workload is client-only", + namespace: "default", + serviceAccount: "default", + want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := extKeyUsages(tc.namespace, tc.serviceAccount) + if !slices.Equal(got, tc.want) { + t.Errorf("extKeyUsages(%q, %q) = %v, want %v", tc.namespace, tc.serviceAccount, got, tc.want) + } + }) + } +} diff --git a/internal/ateapiauth/client.go b/internal/ateapiauth/client.go index c88a16e22..2a1957484 100644 --- a/internal/ateapiauth/client.go +++ b/internal/ateapiauth/client.go @@ -75,7 +75,7 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { return nil, err } tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS13, RootCAs: pool, ServerName: cfg.ServerName, GetClientCertificate: credbundle.ClientLoader(cfg.ClientCredBundle), @@ -93,7 +93,7 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { return nil, err } tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS13, RootCAs: pool, ServerName: cfg.ServerName, } diff --git a/internal/ateapiauth/client_test.go b/internal/ateapiauth/client_test.go index 512df7f42..74c30b823 100644 --- a/internal/ateapiauth/client_test.go +++ b/internal/ateapiauth/client_test.go @@ -34,8 +34,9 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/emptypb" ) func TestDialOptionsRequiresCAFile(t *testing.T) { @@ -82,8 +83,9 @@ func TestDialOptionsMTLSHandshake(t *testing.T) { Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: caPool, - MinVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS13, }))) + healthpb.RegisterHealthServer(srv, health.NewServer()) lis, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) @@ -103,32 +105,30 @@ func TestDialOptionsMTLSHandshake(t *testing.T) { if err != nil { t.Fatalf("DialOptions() error = %v", err) } - // Unimplemented proves the TLS handshake and HTTP/2 setup succeeded - // and the RPC reached the server (which has no services registered). - if code := invokeCode(ctx, t, lis.Addr().String(), opts); code != codes.Unimplemented { - t.Fatalf("Invoke() code = %v, want %v", code, codes.Unimplemented) + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code != codes.OK { + t.Fatalf("health check code = %v, want %v", code, codes.OK) } }) t.Run("without client cert is rejected", func(t *testing.T) { opts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ RootCAs: caPool, - MinVersion: tls.VersionTLS12, + MinVersion: tls.VersionTLS13, }))} - if code := invokeCode(ctx, t, lis.Addr().String(), opts); code == codes.Unimplemented { - t.Fatalf("Invoke() code = %v, want handshake failure", code) + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code == codes.OK { + t.Fatalf("health check code = %v, want handshake failure", code) } }) } -func invokeCode(ctx context.Context, t *testing.T, target string, opts []grpc.DialOption) codes.Code { +func healthCheckCode(ctx context.Context, t *testing.T, target string, opts []grpc.DialOption) codes.Code { t.Helper() conn, err := grpc.NewClient(target, opts...) if err != nil { t.Fatalf("grpc.NewClient() error = %v", err) } defer conn.Close() - err = conn.Invoke(ctx, "/test.NoSuchService/Ping", &emptypb.Empty{}, &emptypb.Empty{}) + _, err = healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}) return status.Code(err) } diff --git a/vendor/google.golang.org/grpc/health/client.go b/vendor/google.golang.org/grpc/health/client.go new file mode 100644 index 000000000..740745c45 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/client.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "fmt" + "io" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/status" +) + +var ( + backoffStrategy = backoff.DefaultExponential + backoffFunc = func(ctx context.Context, retries int) bool { + d := backoffStrategy.Backoff(retries) + timer := time.NewTimer(d) + select { + case <-timer.C: + return true + case <-ctx.Done(): + timer.Stop() + return false + } + } +) + +func init() { + internal.HealthCheckFunc = clientHealthCheck +} + +const healthCheckMethod = "/grpc.health.v1.Health/Watch" + +// This function implements the protocol defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +func clientHealthCheck(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), service string) error { + tryCnt := 0 + +retryConnection: + for { + // Backs off if the connection has failed in some way without receiving a message in the previous retry. + if tryCnt > 0 && !backoffFunc(ctx, tryCnt-1) { + return nil + } + tryCnt++ + + if ctx.Err() != nil { + return nil + } + setConnectivityState(connectivity.Connecting, nil) + rawS, err := newStream(healthCheckMethod) + if err != nil { + continue retryConnection + } + + s, ok := rawS.(grpc.ClientStream) + // Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes. + if !ok { + setConnectivityState(connectivity.Ready, nil) + return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS) + } + + if err = s.SendMsg(&healthpb.HealthCheckRequest{Service: service}); err != nil && err != io.EOF { + // Stream should have been closed, so we can safely continue to create a new stream. + continue retryConnection + } + s.CloseSend() + + resp := new(healthpb.HealthCheckResponse) + for { + err = s.RecvMsg(resp) + + // Reports healthy for the LBing purposes if health check is not implemented in the server. + if status.Code(err) == codes.Unimplemented { + setConnectivityState(connectivity.Ready, nil) + return err + } + + // Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED. + if err != nil { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but received health check RPC error: %v", err)) + continue retryConnection + } + + // As a message has been received, removes the need for backoff for the next retry by resetting the try count. + tryCnt = 0 + if resp.Status == healthpb.HealthCheckResponse_SERVING { + setConnectivityState(connectivity.Ready, nil) + } else { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but health check failed. status=%s", resp.Status)) + } + } + } +} diff --git a/vendor/google.golang.org/grpc/health/logging.go b/vendor/google.golang.org/grpc/health/logging.go new file mode 100644 index 000000000..83c6acf55 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/logging.go @@ -0,0 +1,23 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import "google.golang.org/grpc/grpclog" + +var logger = grpclog.Component("health_service") diff --git a/vendor/google.golang.org/grpc/health/producer.go b/vendor/google.golang.org/grpc/health/producer.go new file mode 100644 index 000000000..f938e5790 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/producer.go @@ -0,0 +1,106 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/status" +) + +func init() { + producerBuilderSingleton = &producerBuilder{} + internal.RegisterClientHealthCheckListener = registerClientSideHealthCheckListener +} + +type producerBuilder struct{} + +var producerBuilderSingleton *producerBuilder + +// Build constructs and returns a producer and its cleanup function. +func (*producerBuilder) Build(cci any) (balancer.Producer, func()) { + p := &healthServiceProducer{ + cc: cci.(grpc.ClientConnInterface), + cancel: func() {}, + } + return p, func() { + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + } +} + +type healthServiceProducer struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + cc grpc.ClientConnInterface + + mu sync.Mutex + cancel func() +} + +// registerClientSideHealthCheckListener accepts a listener to provide server +// health state via the health service. +func registerClientSideHealthCheckListener(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) func() { + pr, closeFn := sc.GetOrBuildProducer(producerBuilderSingleton) + p := pr.(*healthServiceProducer) + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + if listener == nil { + return closeFn + } + + ctx, cancel := context.WithCancel(ctx) + p.cancel = cancel + + go p.startHealthCheck(ctx, sc, serviceName, listener) + return closeFn +} + +func (p *healthServiceProducer) startHealthCheck(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) { + newStream := func(method string) (any, error) { + return p.cc.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, method) + } + + setConnectivityState := func(state connectivity.State, err error) { + listener(balancer.SubConnState{ + ConnectivityState: state, + ConnectionError: err, + }) + } + + // Call the function through the internal variable as tests use it for + // mocking. + err := internal.HealthCheckFunc(ctx, newStream, setConnectivityState, serviceName) + if err == nil { + return + } + if status.Code(err) == codes.Unimplemented { + logger.Errorf("Subchannel health check is unimplemented at server side, thus health check is disabled for SubConn %p", sc) + } else { + logger.Errorf("Health checking failed for SubConn %p: %v", sc, err) + } +} diff --git a/vendor/google.golang.org/grpc/health/server.go b/vendor/google.golang.org/grpc/health/server.go new file mode 100644 index 000000000..d8eebe99d --- /dev/null +++ b/vendor/google.golang.org/grpc/health/server.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package health provides a service that exposes server's health and it must be +// imported to enable support for client-side health checks. +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc/codes" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +const ( + // maxAllowedServices defines the maximum number of resources a List + // operation can return. An error is returned if the number of services + // exceeds this limit. + maxAllowedServices = 100 +) + +// Server implements `service Health`. +type Server struct { + healthgrpc.UnimplementedHealthServer + mu sync.RWMutex + // If shutdown is true, it's expected all serving status is NOT_SERVING, and + // will stay in NOT_SERVING. + shutdown bool + // statusMap stores the serving status of the services this Server monitors. + statusMap map[string]healthpb.HealthCheckResponse_ServingStatus + updates map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus +} + +// NewServer returns a new Server. +func NewServer() *Server { + return &Server{ + statusMap: map[string]healthpb.HealthCheckResponse_ServingStatus{"": healthpb.HealthCheckResponse_SERVING}, + updates: make(map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus), + } +} + +// Check implements `service Health`. +func (s *Server) Check(_ context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if servingStatus, ok := s.statusMap[in.Service]; ok { + return &healthpb.HealthCheckResponse{ + Status: servingStatus, + }, nil + } + return nil, status.Error(codes.NotFound, "unknown service") +} + +// List implements `service Health`. +func (s *Server) List(_ context.Context, _ *healthpb.HealthListRequest) (*healthpb.HealthListResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if len(s.statusMap) > maxAllowedServices { + return nil, status.Errorf(codes.ResourceExhausted, "server health list exceeds maximum capacity: %d", maxAllowedServices) + } + + statusMap := make(map[string]*healthpb.HealthCheckResponse, len(s.statusMap)) + for k, v := range s.statusMap { + statusMap[k] = &healthpb.HealthCheckResponse{Status: v} + } + + return &healthpb.HealthListResponse{Statuses: statusMap}, nil +} + +// Watch implements `service Health`. +func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + service := in.Service + // update channel is used for getting service status updates. + update := make(chan healthpb.HealthCheckResponse_ServingStatus, 1) + s.mu.Lock() + // Puts the initial status to the channel. + if servingStatus, ok := s.statusMap[service]; ok { + update <- servingStatus + } else { + update <- healthpb.HealthCheckResponse_SERVICE_UNKNOWN + } + + // Registers the update channel to the correct place in the updates map. + if _, ok := s.updates[service]; !ok { + s.updates[service] = make(map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus) + } + s.updates[service][stream] = update + defer func() { + s.mu.Lock() + delete(s.updates[service], stream) + s.mu.Unlock() + }() + s.mu.Unlock() + + var lastSentStatus healthpb.HealthCheckResponse_ServingStatus = -1 + for { + select { + // Status updated. Sends the up-to-date status to the client. + case servingStatus := <-update: + if lastSentStatus == servingStatus { + continue + } + lastSentStatus = servingStatus + err := stream.Send(&healthpb.HealthCheckResponse{Status: servingStatus}) + if err != nil { + return status.Error(codes.Canceled, "Stream has ended.") + } + // Context done. Removes the update channel from the updates map. + case <-stream.Context().Done(): + return status.Error(codes.Canceled, "Stream has ended.") + } + } +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + defer s.mu.Unlock() + if s.shutdown { + logger.Infof("health: status changing for %s to %v is ignored because health service is shutdown", service, servingStatus) + return + } + + s.setServingStatusLocked(service, servingStatus) +} + +func (s *Server) setServingStatusLocked(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.statusMap[service] = servingStatus + for _, update := range s.updates[service] { + // Clears previous updates, that are not sent to the client, from the channel. + // This can happen if the client is not reading and the server gets flow control limited. + select { + case <-update: + default: + } + // Puts the most recent update to the channel. + update <- servingStatus + } +} + +// Shutdown sets all serving status to NOT_SERVING, and configures the server to +// ignore all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Shutdown() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = true + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_NOT_SERVING) + } +} + +// Resume sets all serving status to SERVING, and configures the server to +// accept all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Resume() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = false + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_SERVING) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 13877b16e..5a18bf468 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1031,6 +1031,7 @@ google.golang.org/grpc/experimental/opentelemetry google.golang.org/grpc/experimental/stats google.golang.org/grpc/grpclog google.golang.org/grpc/grpclog/internal +google.golang.org/grpc/health google.golang.org/grpc/health/grpc_health_v1 google.golang.org/grpc/internal google.golang.org/grpc/internal/admin From 620a215ddc051d74f8f81e88401d256244de7047 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Fri, 17 Jul 2026 12:58:34 -0700 Subject: [PATCH 7/9] replaced 'annoymous' with authn failure by switching ate apiserver to use a chained authenticator --- .github/workflows/pr-workflow.yaml | 8 +- cmd/ateapi/main.go | 18 +-- cmd/atecontroller/main.go | 14 +- cmd/atenet/internal/router/cmd.go | 8 +- cmd/atenet/internal/router/config.go | 11 +- cmd/atenet/internal/router/router.go | 8 +- hack/install-ate.sh | 38 +++--- hack/run-microvm-demo.sh | 20 +-- internal/ateapiauth/client.go | 78 +++++------ internal/ateapiauth/client_test.go | 95 +++++++++++--- internal/ateapiauth/server.go | 117 ++++++----------- internal/ateapiauth/server_test.go | 123 +++++++++++------- internal/ateclient/builder.go | 64 +++------ internal/ateclient/builder_test.go | 36 +++-- .../kustomization.yaml | 33 +++-- .../{jwt => token-client}/kustomization.yaml | 31 +++-- .../{jwt => token-client}/patches.yaml | 0 17 files changed, 350 insertions(+), 352 deletions(-) rename manifests/ate-install/{kind-jwt => kind-token-client}/kustomization.yaml (56%) rename manifests/ate-install/{jwt => token-client}/kustomization.yaml (60%) rename manifests/ate-install/{jwt => token-client}/patches.yaml (100%) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 5a6c99eec..3da77e517 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -43,8 +43,8 @@ jobs: strategy: fail-fast: false matrix: - auth-mode: [mtls, jwt] - name: e2e-test (${{ matrix.auth-mode }}) + ateapi-client-auth: [cert, token] + name: e2e-test (${{ matrix.ateapi-client-auth }}) steps: - name: Checkout uses: actions/checkout@v5 @@ -88,11 +88,11 @@ jobs: - name: Create cluster run: hack/create-kind-cluster.sh - name: Install Agent Substrate - run: hack/install-ate-kind.sh --deploy-ate-system --auth-mode=${{ matrix.auth-mode }} + run: hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and applies the # counter-microvm demo onto the control plane installed above. - run: hack/run-microvm-demo-kind.sh --auth-mode=${{ matrix.auth-mode }} + run: hack/run-microvm-demo-kind.sh --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Wait for micro-VM golden snapshot diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 1ad88c38b..0742e83d8 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -74,7 +74,6 @@ var ( ateletCACerts = pflag.String("atelet-ca-certs", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify atelet serving certificates.") showVersion = pflag.Bool("version", false, "Print version and exit.") - authMode = pflag.String("auth-mode", "mtls", "Auth mode for incoming gRPC: mtls|jwt. 'mtls' (default) relies on transport-level mTLS for client identity. 'jwt' additionally requires a Kubernetes ServiceAccount Bearer token on every RPC. Substrate will drop support for JWT auth mode once the Pod Certificates feature is enabled by default in the minimum supported Kubernetes version.") clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.") ) @@ -105,11 +104,6 @@ func main() { loadFlagsFromEnv() logFlagValues(ctx) - authModeParsed, err := ateapiauth.ParseMode(*authMode) - if err != nil { - serverboot.Fatal(ctx, "Invalid --auth-mode", err) - } - redisClient, err := connectRedis(ctx) if err != nil { serverboot.Fatal(ctx, "Failed to set up Redis/Valkey", err) @@ -157,9 +151,6 @@ func main() { sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, ateletDialer, clientset) jwtIssuerDiscoveryClient := buildK8sServiceAccountIssuerDiscoveryClient(ctx, *clientJWTCAFile, *clientJWTIssuer) - if authModeParsed == ateapiauth.ModeJWT && jwtIssuerDiscoveryClient == nil { - serverboot.Fatal(ctx, "JWT auth mode requires a Kubernetes ServiceAccount issuer discovery client", fmt.Errorf("client JWT issuer %q is not usable for discovery", *clientJWTIssuer)) - } sessionIdentitySrv := sessionidentity.New(*clientJWTIssuer, *clientJWTAudience, *sessionIDJWTPoolFile, *sessionIDCAPoolFile, *workerpoolCACerts, jwtIssuerDiscoveryClient) @@ -170,7 +161,6 @@ func main() { } authCfg := ateapiauth.ServerConfig{ - Mode: authModeParsed, VerifyBearerToken: func(ctx context.Context, bearer string) (string, error) { claims, err := k8sjwt.Verify(ctx, jwtIssuerDiscoveryClient, bearer, *clientJWTIssuer, *clientJWTAudience, time.Now()) if err != nil { @@ -244,7 +234,6 @@ func logFlagValues(ctx context.Context) { slog.String("session-id-jwt-pool", *sessionIDJWTPoolFile), slog.String("session-id-ca-pool", *sessionIDCAPoolFile), slog.String("workerpool-ca-certs", *workerpoolCACerts), - slog.String("auth-mode", *authMode), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), slog.String("atelet-ca-certs", *ateletCACerts), ) @@ -372,8 +361,11 @@ func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, er } return credentials.NewTLS(&tls.Config{ GetCertificate: credbundle.Loader(*grpcServerCredBundle), - ClientAuth: tls.VerifyClientCertIfGiven, - ClientCAs: clientCAs, + // Client certs stay optional at the transport level: certless + // clients such as kubectl-ate authenticate with a Bearer token in the + // ateapiauth interceptor. + ClientAuth: tls.VerifyClientCertIfGiven, + ClientCAs: clientCAs, }), nil } diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index 48069eab3..fc9edc098 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -39,11 +39,11 @@ var ( ateAPIConnSpec = pflag.String("ateapi-conn-spec", "dns:///api.ate-system.svc:443", "") - ateapiAuthMode = pflag.String("ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) verifies the server cert and presents the pod-projected client certificate. 'jwt' verifies the server cert and sends a Bearer SA token.") ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") - ateapiTokenFile = pflag.String("ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") - ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required for mtls.") + ateapiTokenAuth = pflag.Bool("ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") + ateapiTokenFile = pflag.String("ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") + ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required unless --ateapi-use-token-auth is set, ignored otherwise.") ) func init() { @@ -55,14 +55,8 @@ func main() { pflag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) - mode, err := ateapiauth.ParseMode(*ateapiAuthMode) - if err != nil { - setupLog.Error(err, "invalid --ateapi-auth") - os.Exit(1) - } - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: mode, + UseTokenAuth: *ateapiTokenAuth, CAFile: *ateapiCAFile, ServerName: *ateapiServerName, TokenFile: *ateapiTokenFile, diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 873731929..3cddd048d 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -19,8 +19,6 @@ import ( "time" "github.com/spf13/cobra" - - "github.com/agent-substrate/substrate/internal/ateapiauth" ) func NewRouterCmd() *cobra.Command { @@ -57,11 +55,11 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)") - cmd.Flags().StringVar(&cfg.Auth.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) verifies the server cert (--ateapi-ca-file) and presents the pod-projected client certificate (--ateapi-client-cert). 'jwt' verifies the server cert and sends a Bearer SA token.") cmd.Flags().StringVar(&cfg.Auth.AteapiCAFile, "ateapi-ca-file", "", "PEM file with CAs trusted to verify the ateapi server cert. Required.") - cmd.Flags().StringVar(&cfg.Auth.AteapiClientCertPath, "ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required for mtls.") + cmd.Flags().StringVar(&cfg.Auth.AteapiClientCertPath, "ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required unless --ateapi-use-token-auth is set, ignored otherwise.") cmd.Flags().StringVar(&cfg.Auth.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") - cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") + cmd.Flags().BoolVar(&cfg.Auth.AteapiUseTokenAuth, "ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") + cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") return cmd } diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 2b7d3732b..83893ea9b 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -19,12 +19,13 @@ import ( ) // authConfig holds the router's client-auth settings for dialing ateapi. -// AteapiCAFile verifies ateapi's serving cert in both modes (the servicedns -// trust bundle in-cluster); in mtls mode the router additionally presents -// AteapiClientCertPath (the podidentity credential bundle) as its client -// cert, in jwt mode it sends a Bearer token from AteapiTokenFile instead. +// AteapiCAFile always verifies ateapi's serving cert (the servicedns trust +// bundle in-cluster). By default the router presents AteapiClientCertPath +// (the podidentity credential bundle) as its client cert; with +// AteapiUseTokenAuth it sends a Bearer token from AteapiTokenFile instead and +// the cert path is ignored. type authConfig struct { - AteapiAuthMode string + AteapiUseTokenAuth bool AteapiCAFile string AteapiClientCertPath string AteapiServerName string diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index b28372a81..61cfb0cf4 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -158,12 +158,8 @@ func (s *RouterServer) Run(ctx context.Context) error { go serverboot.StartMetricsServer(ctx, serverboot.MetricsServerOptions{Addr: s.cfg.MetricsAddr}) - authMode, err := ateapiauth.ParseMode(s.cfg.Auth.AteapiAuthMode) - if err != nil { - return fmt.Errorf("invalid --ateapi-auth: %w", err) - } dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: authMode, + UseTokenAuth: s.cfg.Auth.AteapiUseTokenAuth, CAFile: s.cfg.Auth.AteapiCAFile, ServerName: s.cfg.Auth.AteapiServerName, TokenFile: s.cfg.Auth.AteapiTokenFile, @@ -180,7 +176,7 @@ func (s *RouterServer) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to establish grpc channel to ateapi client: %w", err) } - slog.InfoContext(ctx, "Connecting to ateapi", slog.String("address", s.cfg.AteapiAddr), slog.String("auth", string(authMode))) + slog.InfoContext(ctx, "Connecting to ateapi", slog.String("address", s.cfg.AteapiAddr), slog.Bool("use-api-token-auth", s.cfg.Auth.AteapiUseTokenAuth)) s.apiClient = ateapipb.NewControlClient(conn) slog.InfoContext(ctx, "Starting substrate router subsystem", slog.Bool("standalone", s.cfg.Standalone)) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index c0c5c02f2..6b87c8c59 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -63,7 +63,7 @@ function usage() { echo " --deploy-ate-system Deploy core system (CRDs, atelet, apiserver)" echo " --delete-ate-system Delete core system" echo " --delete-all Delete core system and all registered demos" - echo " --auth-mode=mtls|jwt Select ateapi auth mode for --deploy-ate-system (default: mtls)" + echo " --ateapi-client-auth=cert|token Select how in-cluster clients authenticate to ateapi for --deploy-ate-system (default: cert; the server always accepts both)" echo "" echo "Infrastructure components:" echo "" @@ -133,26 +133,26 @@ run_ko() { esac } -ate_auth_mode() { - case "${ATE_API_AUTH_MODE:-mtls}" in - mtls|jwt) - echo "${ATE_API_AUTH_MODE:-mtls}" +ateapi_client_auth() { + case "${ATE_ATEAPI_CLIENT_AUTH:-cert}" in + cert|token) + echo "${ATE_ATEAPI_CLIENT_AUTH:-cert}" ;; *) - echo "Error: ATE_API_AUTH_MODE must be mtls or jwt, got '${ATE_API_AUTH_MODE}'" >&2 + echo "Error: ATE_ATEAPI_CLIENT_AUTH must be cert or token, got '${ATE_ATEAPI_CLIENT_AUTH}'" >&2 exit 1 ;; esac } render_ate_system_manifests() { - local auth_mode="" - auth_mode="$(ate_auth_mode)" + local client_auth="" + client_auth="$(ateapi_client_auth)" - if [[ "${auth_mode}" == "jwt" ]]; then - local overlay="manifests/ate-install/jwt" + if [[ "${client_auth}" == "token" ]]; then + local overlay="manifests/ate-install/token-client" if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - overlay="manifests/ate-install/kind-jwt" + overlay="manifests/ate-install/kind-token-client" fi kubectl kustomize "${overlay}" --load-restrictor LoadRestrictionsNone | run_ko resolve -f - return @@ -551,13 +551,13 @@ BENCHMARK_WORKER_COUNT=1 prescan_args=("$@") for ((i = 0; i < ${#prescan_args[@]}; i++)); do case "${prescan_args[i]}" in - --auth-mode=*) ATE_API_AUTH_MODE="${prescan_args[i]#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${prescan_args[i]#*=}" ;; + --ateapi-client-auth) if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi - ATE_API_AUTH_MODE="${prescan_args[$((i + 1))]}" + ATE_ATEAPI_CLIENT_AUTH="${prescan_args[$((i + 1))]}" ;; --benchmark-worker-count) BENCHMARK_WORKER_COUNT="${prescan_args[i+1]:-1}" @@ -582,14 +582,14 @@ while [[ "$#" -gt 0 ]]; do done case $1 in - --auth-mode=*) ATE_API_AUTH_MODE="${1#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${1#*=}" ;; + --ateapi-client-auth) shift if [[ "$#" -eq 0 ]]; then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi - ATE_API_AUTH_MODE="$1" + ATE_ATEAPI_CLIENT_AUTH="$1" ;; --deploy-ate-system) deploy_ate_system ;; diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 2e29f6ba6..1f92da24a 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -48,18 +48,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -ATE_API_AUTH_MODE="${ATE_API_AUTH_MODE:-mtls}" +ATE_ATEAPI_CLIENT_AUTH="${ATE_ATEAPI_CLIENT_AUTH:-cert}" while [[ $# -gt 0 ]]; do case "$1" in - --auth-mode=*) ATE_API_AUTH_MODE="${1#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${1#*=}" ;; + --ateapi-client-auth) if [[ $# -lt 2 ]]; then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi shift - ATE_API_AUTH_MODE="$1" + ATE_ATEAPI_CLIENT_AUTH="$1" ;; *) echo "Error: unknown argument $1" >&2 @@ -69,10 +69,10 @@ while [[ $# -gt 0 ]]; do shift done -case "${ATE_API_AUTH_MODE}" in - mtls|jwt) ;; +case "${ATE_ATEAPI_CLIENT_AUTH}" in + cert|token) ;; *) - echo "Error: --auth-mode must be mtls or jwt, got '${ATE_API_AUTH_MODE}'" >&2 + echo "Error: --ateapi-client-auth must be cert or token, got '${ATE_ATEAPI_CLIENT_AUTH}'" >&2 exit 1 ;; esac @@ -131,10 +131,10 @@ fi log "Deploying the ate control plane (--deploy-ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" fi # --- 4. apply the demo ------------------------------------------------------ diff --git a/internal/ateapiauth/client.go b/internal/ateapiauth/client.go index 2a1957484..aed3b5d71 100644 --- a/internal/ateapiauth/client.go +++ b/internal/ateapiauth/client.go @@ -32,30 +32,32 @@ const ( DefaultServiceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" ) -// ClientConfig configures how to dial the ateapi gRPC server. -// -// - Mode=ModeMTLS: mutual TLS. Validates the server cert against CAFile and -// presents the client certificate from ClientCredBundle, re-read on every -// handshake so in-place pod-certificate rotations are picked up. -// - Mode=ModeJWT: validates the server cert against CAFile, sends a Bearer -// token from TokenFile as per-RPC credentials. +// ClientConfig configures how to dial the ateapi gRPC server. The server +// cert is always validated against CAFile. UseTokenAuth selects the client +// credential: a client certificate from ClientCredBundle (mutual TLS, +// re-read on every handshake so in-place pod-certificate rotations are +// picked up) by default, or a Bearer token from TokenFile sent as per-RPC +// credentials. The path not selected is ignored. type ClientConfig struct { - Mode Mode + // UseTokenAuth authenticates with the Bearer token from TokenFile instead + // of the client certificate from ClientCredBundle. + UseTokenAuth bool // CAFile is a PEM file containing CA certs that sign the server cert. - // Required in all modes. + // Required. CAFile string // ServerName overrides SNI / hostname verification. Optional. ServerName string // TokenFile is a path to a Kubernetes projected ServiceAccount token used - // as a Bearer credential. Required for ModeJWT. + // as a Bearer credential. Required when UseTokenAuth is set, ignored + // otherwise. TokenFile string // ClientCredBundle is a PEM file containing the client certificate chain - // and PKCS8 private key presented to the server. Required for ModeMTLS - // and ignored for ModeJWT. + // and PKCS8 private key presented to the server. Required unless + // UseTokenAuth is set, ignored otherwise. ClientCredBundle string } @@ -65,46 +67,32 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { if cfg.CAFile == "" { return nil, fmt.Errorf("ateapiauth: CAFile is required") } - switch cfg.Mode { - case "", ModeMTLS: - if cfg.ClientCredBundle == "" { - return nil, fmt.Errorf("ateapiauth: mtls mode requires ClientCredBundle") - } - pool, err := loadCAPool(cfg.CAFile) - if err != nil { - return nil, err - } - tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS13, - RootCAs: pool, - ServerName: cfg.ServerName, - GetClientCertificate: credbundle.ClientLoader(cfg.ClientCredBundle), - } - return []grpc.DialOption{ - grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), - }, nil - - case ModeJWT: + if cfg.UseTokenAuth { if cfg.TokenFile == "" { - return nil, fmt.Errorf("ateapiauth: jwt mode requires TokenFile") - } - pool, err := loadCAPool(cfg.CAFile) - if err != nil { - return nil, err - } - tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS13, - RootCAs: pool, - ServerName: cfg.ServerName, + return nil, fmt.Errorf("ateapiauth: token auth requires a token file") } + } else if cfg.ClientCredBundle == "" { + return nil, fmt.Errorf("ateapiauth: a client credential bundle (mTLS) is required unless token auth is enabled") + } + pool, err := loadCAPool(cfg.CAFile) + if err != nil { + return nil, err + } + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: pool, + ServerName: cfg.ServerName, + } + if cfg.UseTokenAuth { return []grpc.DialOption{ grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), grpc.WithPerRPCCredentials(&fileTokenCreds{path: cfg.TokenFile}), }, nil - - default: - return nil, fmt.Errorf("ateapiauth: unknown client mode %q", cfg.Mode) } + tlsCfg.GetClientCertificate = credbundle.ClientLoader(cfg.ClientCredBundle) + return []grpc.DialOption{ + grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), + }, nil } func loadCAPool(caFile string) (*x509.CertPool, error) { diff --git a/internal/ateapiauth/client_test.go b/internal/ateapiauth/client_test.go index 74c30b823..9109699d2 100644 --- a/internal/ateapiauth/client_test.go +++ b/internal/ateapiauth/client_test.go @@ -28,6 +28,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "testing" "time" @@ -36,16 +37,17 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) func TestDialOptionsRequiresCAFile(t *testing.T) { - for _, mode := range []Mode{ModeMTLS, ModeJWT} { - t.Run(string(mode), func(t *testing.T) { - _, err := DialOptions(ClientConfig{ - Mode: mode, - TokenFile: "token", - }) + for name, cfg := range map[string]ClientConfig{ + "token": {UseTokenAuth: true, TokenFile: "token"}, + "cert": {ClientCredBundle: "bundle.pem"}, + } { + t.Run(name, func(t *testing.T) { + _, err := DialOptions(cfg) if err == nil { t.Fatalf("DialOptions() error = nil, want error") } @@ -53,20 +55,25 @@ func TestDialOptionsRequiresCAFile(t *testing.T) { } } -func TestDialOptionsMTLSRequiresClientCredBundle(t *testing.T) { - _, err := DialOptions(ClientConfig{ - Mode: ModeMTLS, - CAFile: "ca.pem", - }) - if err == nil { - t.Fatalf("DialOptions() error = nil, want error for missing ClientCredBundle") +func TestDialOptionsRequiresModeCredential(t *testing.T) { + for name, cfg := range map[string]ClientConfig{ + "cert mode without bundle": {CAFile: "ca.pem"}, + "token mode without token": {CAFile: "ca.pem", UseTokenAuth: true}, + "cert path does not satisfy token mode": {CAFile: "ca.pem", UseTokenAuth: true, ClientCredBundle: "bundle.pem"}, + } { + t.Run(name, func(t *testing.T) { + _, err := DialOptions(cfg) + if err == nil { + t.Fatalf("DialOptions() error = nil, want error") + } + }) } } // TestDialOptionsMTLSHandshake dials a server that requires and verifies // client certificates — the configuration ateapi will move to — and checks -// that DialOptions in mtls mode completes the handshake, while a -// certificate-less client is rejected. +// that DialOptions with a client credential bundle completes the handshake, +// while a certificate-less client is rejected. func TestDialOptionsMTLSHandshake(t *testing.T) { ca := newTestCA(t) dir := t.TempDir() @@ -97,10 +104,11 @@ func TestDialOptionsMTLSHandshake(t *testing.T) { defer cancel() t.Run("with client cert", func(t *testing.T) { + // TokenFile is ignored in cert mode (the default). opts, err := DialOptions(ClientConfig{ - Mode: ModeMTLS, CAFile: caFile, ClientCredBundle: clientBundle, + TokenFile: filepath.Join(dir, "does-not-exist-token"), }) if err != nil { t.Fatalf("DialOptions() error = %v", err) @@ -121,6 +129,61 @@ func TestDialOptionsMTLSHandshake(t *testing.T) { }) } +// TestDialOptionsTokenSendsBearer dials a server that accepts certless +// clients — ateapi's current configuration — and checks that DialOptions +// with a token file attaches the token as an `authorization: Bearer` header +// on every RPC. +func TestDialOptionsTokenSendsBearer(t *testing.T) { + ca := newTestCA(t) + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.pem") + writeFile(t, caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.certDER})) + tokenFile := filepath.Join(dir, "token") + writeFile(t, tokenFile, []byte("test-token\n")) + + gotAuth := make(chan string, 1) + srv := grpc.NewServer( + grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{ca.issueServerCert(t)}, + ClientAuth: tls.VerifyClientCertIfGiven, + MinVersion: tls.VersionTLS13, + })), + grpc.UnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, _ := metadata.FromIncomingContext(ctx) + gotAuth <- strings.Join(md.Get("authorization"), ",") + return handler(ctx, req) + }), + ) + healthpb.RegisterHealthServer(srv, health.NewServer()) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + go srv.Serve(lis) + defer srv.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // ClientCredBundle stays set (as the base manifests leave it) but is + // ignored in token mode — it doesn't even need to exist on disk. + opts, err := DialOptions(ClientConfig{ + CAFile: caFile, + UseTokenAuth: true, + TokenFile: tokenFile, + ClientCredBundle: filepath.Join(dir, "does-not-exist.pem"), + }) + if err != nil { + t.Fatalf("DialOptions() error = %v", err) + } + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code != codes.OK { + t.Fatalf("health check code = %v, want %v", code, codes.OK) + } + if got, want := <-gotAuth, "Bearer test-token"; got != want { + t.Errorf("authorization header = %q, want %q", got, want) + } +} + func healthCheckCode(ctx context.Context, t *testing.T, target string, opts []grpc.DialOption) codes.Code { t.Helper() conn, err := grpc.NewClient(target, opts...) diff --git a/internal/ateapiauth/server.go b/internal/ateapiauth/server.go index 3baaf476f..b2891c9da 100644 --- a/internal/ateapiauth/server.go +++ b/internal/ateapiauth/server.go @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ateapiauth adds optional Kubernetes ServiceAccount JWT -// authentication on top of the ateapi gRPC server, and a matching client -// dial helper. It does not replace the existing TLS / mTLS path — the -// server's transport credentials still apply unchanged. Set Mode=ModeJWT -// on the server to require an `authorization: Bearer ` header -// on every RPC; Mode=ModeMTLS (the default) leaves identity to the -// transport-layer mTLS credentials. +// Package ateapiauth authenticates clients of the ateapi gRPC server, and +// provides a matching client dial helper. The server interceptor takes +// identity from the transport-layer mTLS credentials when the client +// presented a certificate, and otherwise requires an authorization +// header `Bearer `. Requests with no credentials are rejected. package ateapiauth import ( @@ -36,56 +34,25 @@ import ( "google.golang.org/grpc/status" ) -// Mode selects whether the JWT interceptor enforces a Bearer token. -type Mode string - -const ( - ModeMTLS Mode = "mtls" - ModeJWT Mode = "jwt" -) - -// ParseMode parses a flag value into a Mode, defaulting to ModeMTLS on empty. -// ModeMTLS means identity is established by the transport-layer mTLS -// credentials; the interceptor performs no app-level checks. ModeJWT -// additionally requires a Kubernetes SA Bearer token on every RPC. -func ParseMode(s string) (Mode, error) { - switch Mode(s) { - case "", ModeMTLS: - return ModeMTLS, nil - case ModeJWT: - return ModeJWT, nil - default: - return "", fmt.Errorf("unknown auth mode %q (want mtls|jwt)", s) - } -} - func ValidateServerConfig(cfg ServerConfig) error { - switch cfg.Mode { - case "", ModeMTLS: - return nil - case ModeJWT: - if cfg.VerifyBearerToken == nil { - return fmt.Errorf("jwt mode requires bearer token verifier") - } - return nil - default: - return fmt.Errorf("unknown auth mode %q", cfg.Mode) + if cfg.VerifyBearerToken == nil { + return fmt.Errorf("a bearer token verifier is required") } + return nil } // ServerConfig configures the server-side auth interceptor. type ServerConfig struct { - Mode Mode - // VerifyBearerToken verifies a Bearer token presented by a client and - // returns the authenticated principal's ID (e.g. the JWT subject). - // Required for ModeJWT and ignored for ModeMTLS. + // returns the authenticated principal's ID (e.g. the JWT subject). It + // authenticates clients that did not present a certificate identity + // (e.g. kubectl-ate, which dials without a client certificate). VerifyBearerToken func(context.Context, string) (string, error) } // UnaryServerInterceptor returns a gRPC unary interceptor enforcing cfg. func UnaryServerInterceptor(cfg ServerConfig) grpc.UnaryServerInterceptor { - auth := serverAuthenticatorFor(cfg) + auth := newChainedAuthenticator(cfg) return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { newCtx, err := auth.authenticate(ctx) if err != nil { @@ -97,7 +64,7 @@ func UnaryServerInterceptor(cfg ServerConfig) grpc.UnaryServerInterceptor { // StreamServerInterceptor returns a gRPC stream interceptor enforcing cfg. func StreamServerInterceptor(cfg ServerConfig) grpc.StreamServerInterceptor { - auth := serverAuthenticatorFor(cfg) + auth := newChainedAuthenticator(cfg) return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { newCtx, err := auth.authenticate(ss.Context()) if err != nil { @@ -114,65 +81,63 @@ type wrappedStream struct { func (w *wrappedStream) Context() context.Context { return w.ctx } -type serverAuthenticator interface { - authenticate(context.Context) (context.Context, error) -} - -func serverAuthenticatorFor(cfg ServerConfig) serverAuthenticator { - switch cfg.Mode { - case "", ModeMTLS: - return mtlsServerAuthenticator{} - case ModeJWT: - return jwtServerAuthenticator{ +func newChainedAuthenticator(cfg ServerConfig) chainedServerAuthenticator { + return chainedServerAuthenticator{ + jwt: jwtServerAuthenticator{ verifyBearerToken: cfg.VerifyBearerToken, - } + }, } - - return invalidServerAuthenticator{mode: cfg.Mode} } -type mtlsServerAuthenticator struct{} +// chainedServerAuthenticator first checks mTLS peer, then checks +// a bearer token in the header. +type chainedServerAuthenticator struct { + // jwt authenticates clients that did not present a certificate identity + // (e.g. kubectl-ate, which dials without a client certificate). + jwt jwtServerAuthenticator +} -func (mtlsServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { - pInfo := principal.PrincipalInfo{ - ID: "anonymous", - } +func (a chainedServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { if id, ok := mtlsPeerIdentity(ctx); ok { - pInfo = principal.PrincipalInfo{ + return principal.InjectContext(ctx, principal.PrincipalInfo{ ID: id, Kind: principal.KindMTLS, - } + }), nil } - return principal.InjectContext(ctx, pInfo), nil + return a.jwt.authenticate(ctx) } // mtlsPeerIdentity extracts the client identity (the first URI SAN, a SPIFFE -// ID) from the transport-authenticated peer certificate, if any. +// ID) from the transport-authenticated peer certificate. func mtlsPeerIdentity(ctx context.Context) (string, bool) { p, ok := peer.FromContext(ctx) if !ok || p.AuthInfo == nil { - slog.ErrorContext(ctx, "Authentication failed: no peer or auth info in context.") + slog.DebugContext(ctx, "No mTLS peer identity: no peer or auth info in context.") return "", false } tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) if !ok { - slog.ErrorContext(ctx, "Authentication failed: no TLS info in context.") + slog.DebugContext(ctx, "No mTLS peer identity: no TLS info in context.") return "", false } if len(tlsInfo.State.PeerCertificates) == 0 { - slog.ErrorContext(ctx, "Authentication failed: no peer certificates in TLS info.") + slog.DebugContext(ctx, "No mTLS peer identity: no peer certificates in TLS info.") return "", false } clientCert := tlsInfo.State.PeerCertificates[0] if len(clientCert.URIs) == 0 { - slog.ErrorContext(ctx, "Authentication failed: no URIs in peer certificate.") + slog.DebugContext(ctx, "No mTLS peer identity: no URIs in peer certificate.") return "", false } id := clientCert.URIs[0].String() + if id == "" { + slog.DebugContext(ctx, "No mTLS peer identity: client cert URI is empty string") + return "", false + } slog.InfoContext(ctx, "Authentication successful", slog.String("id", id)) return id, true } @@ -196,14 +161,6 @@ func (a jwtServerAuthenticator) authenticate(ctx context.Context) (context.Conte }), nil } -type invalidServerAuthenticator struct { - mode Mode -} - -func (a invalidServerAuthenticator) authenticate(context.Context) (context.Context, error) { - return nil, status.Errorf(codes.Internal, "invalid auth mode %q", a.mode) -} - func bearerToken(ctx context.Context) (string, bool) { md, ok := metadata.FromIncomingContext(ctx) if !ok { diff --git a/internal/ateapiauth/server_test.go b/internal/ateapiauth/server_test.go index b71e855b4..e9538b5d8 100644 --- a/internal/ateapiauth/server_test.go +++ b/internal/ateapiauth/server_test.go @@ -31,40 +31,14 @@ import ( "google.golang.org/grpc/status" ) -func TestParseMode(t *testing.T) { - cases := []struct { - in string - want Mode - wantErr bool - }{ - {"", ModeMTLS, false}, - {"mtls", ModeMTLS, false}, - {"jwt", ModeJWT, false}, - {"none", "", true}, - {"bogus", "", true}, - } - for _, tc := range cases { - got, err := ParseMode(tc.in) - if (err != nil) != tc.wantErr { - t.Errorf("ParseMode(%q) err=%v wantErr=%v", tc.in, err, tc.wantErr) - } - if !tc.wantErr && got != tc.want { - t.Errorf("ParseMode(%q)=%v want %v", tc.in, got, tc.want) - } - } -} - func TestValidateServerConfig(t *testing.T) { tests := []struct { name string cfg ServerConfig wantErr bool }{ - {name: "mtls zero config", cfg: ServerConfig{Mode: ModeMTLS}}, - {name: "empty mode zero config", cfg: ServerConfig{}}, - {name: "jwt valid", cfg: ServerConfig{Mode: ModeJWT, VerifyBearerToken: func(context.Context, string) (string, error) { return "", nil }}}, - {name: "jwt missing verifier", cfg: ServerConfig{Mode: ModeJWT}, wantErr: true}, - {name: "unknown mode", cfg: ServerConfig{Mode: Mode("bogus")}, wantErr: true}, + {name: "valid", cfg: ServerConfig{VerifyBearerToken: func(context.Context, string) (string, error) { return "", nil }}}, + {name: "missing verifier", cfg: ServerConfig{}, wantErr: true}, } for _, tt := range tests { @@ -77,50 +51,101 @@ func TestValidateServerConfig(t *testing.T) { } } -func TestMTLSServerAuthenticatorPrincipal(t *testing.T) { +func TestChainedServerAuthenticatorPrincipal(t *testing.T) { + const subject = "system:serviceaccount:ate-system:ate-client" spiffeID := &url.URL{Scheme: "spiffe", Host: "ate.dev", Path: "/ns/default/sa/router"} + spiffePeer := func(ctx context.Context) context.Context { + return peer.NewContext(ctx, &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{spiffeID}}}, + }}, + }) + } + withBearer := func(ctx context.Context, token string) context.Context { + return metadata.NewIncomingContext(ctx, metadata.Pairs("authorization", "Bearer "+token)) + } + verifyGoodToken := func(_ context.Context, bearer string) (string, error) { + if bearer != "good-token" { + return "", fmt.Errorf("bad token") + } + return subject, nil + } + tests := []struct { name string ctx context.Context - want principal.PrincipalInfo + // verify is the bearer token verifier; nil means the test fails if + // it is called (the certificate identity must take precedence). + verify func(context.Context, string) (string, error) + want principal.PrincipalInfo + wantCode codes.Code }{ { - name: "no peer", - ctx: context.Background(), - want: principal.PrincipalInfo{ID: "anonymous"}, + name: "no peer and no token", + ctx: context.Background(), + wantCode: codes.Unauthenticated, }, { - name: "peer without certificates", + name: "peer without certificates and no token", ctx: peer.NewContext(context.Background(), &peer.Peer{ AuthInfo: credentials.TLSInfo{}, }), - want: principal.PrincipalInfo{ID: "anonymous"}, + wantCode: codes.Unauthenticated, }, { - name: "certificate without URI SAN", - ctx: peer.NewContext(context.Background(), &peer.Peer{ + name: "no peer with valid bearer", + ctx: withBearer(context.Background(), "good-token"), + verify: verifyGoodToken, + want: principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT}, + wantCode: codes.OK, + }, + { + name: "no peer with invalid bearer", + ctx: withBearer(context.Background(), "bad-token"), + verify: verifyGoodToken, + wantCode: codes.Unauthenticated, + }, + { + name: "certificate without URI SAN with valid bearer", + ctx: withBearer(peer.NewContext(context.Background(), &peer.Peer{ AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ PeerCertificates: []*x509.Certificate{{}}, }}, - }), - want: principal.PrincipalInfo{ID: "anonymous"}, + }), "good-token"), + verify: verifyGoodToken, + want: principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT}, + wantCode: codes.OK, }, { - name: "certificate with SPIFFE URI SAN", - ctx: peer.NewContext(context.Background(), &peer.Peer{ - AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ - PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{spiffeID}}}, - }}, - }), - want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + name: "certificate with SPIFFE URI SAN", + ctx: spiffePeer(context.Background()), + want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + wantCode: codes.OK, + }, + { + name: "certificate takes precedence over bearer", + ctx: withBearer(spiffePeer(context.Background()), "good-token"), + want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + wantCode: codes.OK, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - newCtx, err := (mtlsServerAuthenticator{}).authenticate(tt.ctx) - if err != nil { - t.Fatalf("ModeMTLS should not error: %v", err) + verify := tt.verify + if verify == nil { + verify = func(context.Context, string) (string, error) { + t.Fatal("bearer token verifier called; certificate identity must take precedence") + return "", nil + } + } + auth := newChainedAuthenticator(ServerConfig{VerifyBearerToken: verify}) + newCtx, err := auth.authenticate(tt.ctx) + if code := status.Code(err); code != tt.wantCode { + t.Fatalf("authenticate: code=%v (err=%v), want %v", code, err, tt.wantCode) + } + if tt.wantCode != codes.OK { + return } got, ok := principal.FromContext(newCtx) if !ok { diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index 6e64eb56d..57affaf10 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -21,7 +21,6 @@ import ( "io" "net/http" "os" - "strings" "sync" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -76,7 +75,7 @@ func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, var cli *Client if endpoint != "" { - cli, err = dialDirect(kubeconfigPath, k8sContext, endpoint, traceEnabled) + cli, err = dialDirect(ctx, kubeconfigPath, k8sContext, endpoint, traceEnabled) } else { cli, err = dialPortForward(ctx, kubeconfigPath, k8sContext, traceEnabled) } @@ -90,13 +89,23 @@ func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, return cli, nil } -func dialDirect(kubeconfigPath, k8sContext, endpoint string, traceEnabled bool) (*Client, error) { +func dialDirect(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, traceEnabled bool) (*Client, error) { // Always assume TLS to match production behavior creds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) + clientset, err := NewK8sClientset(kubeconfigPath, k8sContext) + if err != nil { + return nil, fmt.Errorf("failed to create k8s client: %w", err) + } + var opts []grpc.DialOption opts = append(opts, grpc.WithTransportCredentials(creds)) opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) + tokenOpt, err := bearerTokenDialOption(ctx, clientset) + if err != nil { + return nil, err + } + opts = append(opts, tokenOpt) if traceEnabled { opts = append(opts, grpc.WithUnaryInterceptor(newTraceInterceptor())) @@ -211,12 +220,12 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext string, tra var opts []grpc.DialOption opts = append(opts, grpc.WithTransportCredentials(transportCreds)) opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) - jwtOpts, err := jwtDialOptions(ctx, clientset) + tokenOpt, err := bearerTokenDialOption(ctx, clientset) if err != nil { close(stopCh) return nil, err } - opts = append(opts, jwtOpts...) + opts = append(opts, tokenOpt) if traceEnabled { opts = append(opts, grpc.WithUnaryInterceptor(newTraceInterceptor())) @@ -238,15 +247,9 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext string, tra }, nil } -func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grpc.DialOption, error) { - jwtMode, err := isJWTMode(ctx, clientset) - if err != nil { - return nil, err - } - if !jwtMode { - return nil, nil - } - +// bearerTokenDialOption attaches a ServiceAccount token for the ate-client SA +// as per-RPC credentials. +func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset) (grpc.DialOption, error) { expirationSeconds := int64(3600) tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ @@ -261,38 +264,7 @@ func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grp if token.Status.Token == "" { return nil, fmt.Errorf("failed to request ateapi bearer token: token response was empty") } - return []grpc.DialOption{grpc.WithPerRPCCredentials(bearerTokenCreds(token.Status.Token))}, nil -} - -func isJWTMode(ctx context.Context, clientset *kubernetes.Clientset) (bool, error) { - // TODO: Replace deployment introspection with an explicit client-readable - // config file once ateapi auth mode is part of install/runtime config. - deployment, err := clientset.AppsV1().Deployments("ate-system").Get(ctx, "ate-api-server", metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("failed to get ate-api-server deployment: %w", err) - } - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name != "ate-api-server" { - continue - } - return isJWTAuthModeArg(container.Args), nil - } - return false, fmt.Errorf("failed to find ate-api-server container in deployment") -} - -func isJWTAuthModeArg(args []string) bool { - for i, arg := range args { - if arg == "--auth-mode=jwt" { - return true - } - if strings.HasPrefix(arg, "--auth-mode=") { - return strings.TrimPrefix(arg, "--auth-mode=") == "jwt" - } - if arg == "--auth-mode" && i+1 < len(args) { - return args[i+1] == "jwt" - } - } - return false + return grpc.WithPerRPCCredentials(bearerTokenCreds(token.Status.Token)), nil } type bearerTokenCreds string diff --git a/internal/ateclient/builder_test.go b/internal/ateclient/builder_test.go index 62c44feb9..3ed7d31a1 100644 --- a/internal/ateclient/builder_test.go +++ b/internal/ateclient/builder_test.go @@ -14,27 +14,25 @@ package ateclient -import "testing" +import ( + "context" + "testing" +) -func TestIsJWTAuthModeArg(t *testing.T) { - tests := []struct { - name string - args []string - want bool - }{ - {name: "equals jwt", args: []string{"--auth-mode=jwt"}, want: true}, - {name: "split jwt", args: []string{"--auth-mode", "jwt"}, want: true}, - {name: "equals mtls", args: []string{"--auth-mode=mtls"}, want: false}, - {name: "split mtls", args: []string{"--auth-mode", "mtls"}, want: false}, - {name: "missing value", args: []string{"--auth-mode"}, want: false}, - {name: "unrelated", args: []string{"--foo=bar"}, want: false}, +func TestBearerTokenCreds(t *testing.T) { + md, err := bearerTokenCreds("some-token").GetRequestMetadata(context.Background()) + if err != nil { + t.Fatalf("GetRequestMetadata: %v", err) + } + if got, want := md["authorization"], "Bearer some-token"; got != want { + t.Errorf("authorization=%q want %q", got, want) + } + + if _, err := bearerTokenCreds("").GetRequestMetadata(context.Background()); err == nil { + t.Error("GetRequestMetadata with empty token: want error, got nil") } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isJWTAuthModeArg(tt.args); got != tt.want { - t.Fatalf("isJWTAuthModeArg(%v) = %v, want %v", tt.args, got, tt.want) - } - }) + if !bearerTokenCreds("some-token").RequireTransportSecurity() { + t.Error("RequireTransportSecurity() = false, want true") } } diff --git a/manifests/ate-install/kind-jwt/kustomization.yaml b/manifests/ate-install/kind-token-client/kustomization.yaml similarity index 56% rename from manifests/ate-install/kind-jwt/kustomization.yaml rename to manifests/ate-install/kind-token-client/kustomization.yaml index 8769b7a10..d95c6be06 100644 --- a/manifests/ate-install/kind-jwt/kustomization.yaml +++ b/manifests/ate-install/kind-token-client/kustomization.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Overlay: in-cluster clients authenticate to ateapi with a projected +# ServiceAccount token instead of a client certificate. The server always +# accepts both. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -19,17 +22,11 @@ resources: - ../kind patches: - - path: ../jwt/patches.yaml - - target: - group: apps - version: v1 - kind: Deployment - name: ate-api-server - namespace: ate-system - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --auth-mode=jwt + - path: ../token-client/patches.yaml + # --ateapi-use-token-auth switches the client to the projected + # ServiceAccount token below. The base manifests' --ateapi-client-cert is + # removed by index; the preceding `test` op pins the expected value so the + # build fails loudly if the base args are ever reordered. - target: group: apps version: v1 @@ -37,9 +34,14 @@ patches: name: ate-controller namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/1 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/1 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token @@ -50,9 +52,14 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/13 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token diff --git a/manifests/ate-install/jwt/kustomization.yaml b/manifests/ate-install/token-client/kustomization.yaml similarity index 60% rename from manifests/ate-install/jwt/kustomization.yaml rename to manifests/ate-install/token-client/kustomization.yaml index a6331c498..82e39fc7f 100644 --- a/manifests/ate-install/jwt/kustomization.yaml +++ b/manifests/ate-install/token-client/kustomization.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Overlay: in-cluster clients authenticate to ateapi with a projected +# ServiceAccount token instead of a client certificate. The server always +# accepts both. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -26,16 +29,10 @@ resources: patches: - path: patches.yaml - - target: - group: apps - version: v1 - kind: Deployment - name: ate-api-server - namespace: ate-system - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --auth-mode=jwt + # --ateapi-use-token-auth switches the client to the projected + # ServiceAccount token below. The base manifests' --ateapi-client-cert is + # removed by index; the preceding `test` op pins the expected value so the + # build fails loudly if the base args are ever reordered. - target: group: apps version: v1 @@ -43,9 +40,14 @@ patches: name: ate-controller namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/1 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/1 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token @@ -56,9 +58,14 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/13 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token diff --git a/manifests/ate-install/jwt/patches.yaml b/manifests/ate-install/token-client/patches.yaml similarity index 100% rename from manifests/ate-install/jwt/patches.yaml rename to manifests/ate-install/token-client/patches.yaml From 0003f76abd2d45cbef660ead4303643f200570ac Mon Sep 17 00:00:00 2001 From: zoezhao Date: Fri, 17 Jul 2026 14:47:22 -0700 Subject: [PATCH 8/9] Updated ASN Object Identifier and the PodIdentity JSON bytes to match future upstream impl --- cmd/ateapi/internal/controlapi/dialer.go | 30 +- cmd/ateapi/internal/controlapi/dialer_test.go | 22 +- .../podidentitysigner/podidentitysigner.go | 36 +- .../podidentitysigner_test.go | 436 +++++++++++++++++- internal/substratex509/substratex509.go | 131 ++++++ internal/substratex509/substratex509_test.go | 232 ++++++++++ 6 files changed, 817 insertions(+), 70 deletions(-) create mode 100644 internal/substratex509/substratex509.go create mode 100644 internal/substratex509/substratex509_test.go diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index a865cfffa..3cc4fad9f 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -17,13 +17,13 @@ package controlapi import ( "crypto/tls" "crypto/x509" - "encoding/asn1" "errors" "fmt" "os" "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/substratex509" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -34,13 +34,6 @@ import ( var ErrWorkerPodNotFound = errors.New("worker pod not found") -// oidPodUID represents the Pod UID within a certificate extension. -var oidPodUID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 6, 1, 3} - -type customExtensionVal struct { - S string `asn1:"utf8"` -} - // AteletDialer handles gRPC connections to Atelet pods. type AteletDialer struct { workerIndexer cache.Indexer @@ -165,19 +158,18 @@ func verifyAteletServerCert(roots *x509.CertPool, expectedPodUID string) func(tl return fmt.Errorf("verifying server certificate chain: %w", err) } - var foundPodUID customExtensionVal - for _, ext := range leaf.Extensions { - if ext.Id.Equal(oidPodUID) { - if _, err := asn1.Unmarshal(ext.Value, &foundPodUID); err != nil { - return fmt.Errorf("failed to parse oidPodUID extension: %w", err) - } - } + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil { + return fmt.Errorf("failed to parse PodIdentity extension: %w", err) + } + if identity == nil { + return fmt.Errorf("server certificate has no PodIdentity extension, expected pod UID %q", expectedPodUID) } - if foundPodUID.S == "" || expectedPodUID == "" { - return fmt.Errorf("found Pod UID == %q, expected podUID == %q", foundPodUID.S, expectedPodUID) + if identity.PodUID == "" || expectedPodUID == "" { + return fmt.Errorf("found pod UID == %q, expected pod UID == %q", identity.PodUID, expectedPodUID) } - if foundPodUID.S != expectedPodUID { - return fmt.Errorf("pod UID is %q does not match expected %q", foundPodUID.S, expectedPodUID) + if identity.PodUID != expectedPodUID { + return fmt.Errorf("pod UID %q does not match expected %q", identity.PodUID, expectedPodUID) } return nil diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index f12a38bda..f113f9db0 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -21,10 +21,11 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" - "encoding/asn1" "math/big" "testing" "time" + + "github.com/agent-substrate/substrate/internal/substratex509" ) // makeTestCA mints a self-signed CA and returns it along with a pool @@ -59,7 +60,7 @@ func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509.CertP } // makeLeafCert mints a server leaf certificate signed by the given CA. If -// podUID is non-empty it is embedded as the oidPodUID extension. +// podUID is non-empty it is embedded in a PodIdentity extension. func makeLeafCert(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, podUID string) *x509.Certificate { t.Helper() @@ -75,12 +76,19 @@ func makeLeafCert(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, p ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, } if podUID != "" { - podUIDExt, err := asn1.Marshal(customExtensionVal{S: podUID}) + // AddPodIdentityToCertificate requires all fields to be non-empty; + // only PodUID matters to these tests. + err := substratex509.AddPodIdentityToCertificate(&substratex509.PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: podUID, + NodeName: "node-1", + NodeUID: "node-uid", + }, template) if err != nil { - t.Fatalf("marshaling pod UID extension: %v", err) - } - template.ExtraExtensions = []pkix.Extension{ - {Id: oidPodUID, Critical: false, Value: podUIDExt}, + t.Fatalf("adding PodIdentity extension: %v", err) } } der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey) diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go index c8fcf00dc..815d38b6d 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go @@ -19,8 +19,6 @@ import ( "context" "crypto/rand" "crypto/x509" - "crypto/x509/pkix" - "encoding/asn1" "encoding/pem" "fmt" "net/url" @@ -30,6 +28,7 @@ import ( "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/signercontroller" "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -40,9 +39,6 @@ import ( const Name = "podidentity.podcert.ate.dev/identity" const CTBPrefix = "podidentity.podcert.ate.dev:identity:" -// oidPodUID represents the Pod UID within a certificate extension. -var oidPodUID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 6, 1, 3} - // atelet's identity, as installed by manifests/ate-install/atelet.yaml. Pods // running as atelet serve TLS (e.g. to ate-apiserver), so their certs also // carry the serverAuth EKU. @@ -80,10 +76,6 @@ func (h *Impl) SignerName() string { return Name } -type customExtensionVal struct { - S string `asn1:"utf8"` -} - func (h *Impl) DesiredClusterTrustBundles() []*certsv1beta1.ClusterTrustBundle { name := CTBPrefix + "primary-bundle" @@ -148,10 +140,6 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq parent := h.caPool.CAs[0].RootCertificate - podUIDExt, err := asn1.Marshal(customExtensionVal{S: string(pod.UID)}) - if err != nil { - return fmt.Errorf("failed to marshal pod UID %s: %w", pod.UID, err) - } template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, @@ -164,13 +152,21 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq // podidentity CAs). // https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.1 AuthorityKeyId: parent.SubjectKeyId, - ExtraExtensions: []pkix.Extension{ - { - Id: oidPodUID, - Critical: false, - Value: podUIDExt, - }, - }, + } + + // Fields are sourced from the PCR spec (attested by kube-apiserver) rather + // than the Pod object, which lacks the ServiceAccount and Node UIDs. + podIdentity := &substratex509.PodIdentity{ + Namespace: pcr.ObjectMeta.Namespace, + ServiceAccountName: pcr.Spec.ServiceAccountName, + ServiceAccountUID: string(pcr.Spec.ServiceAccountUID), + PodName: pcr.Spec.PodName, + PodUID: string(pcr.Spec.PodUID), + NodeName: string(pcr.Spec.NodeName), + NodeUID: string(pcr.Spec.NodeUID), + } + if err := substratex509.AddPodIdentityToCertificate(podIdentity, template); err != nil { + return fmt.Errorf("while adding pod identity to certificate: %w", err) } subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go index 15300cb3d..6afd1d50c 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go @@ -15,50 +15,438 @@ package podidentitysigner import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" "crypto/x509" + "encoding/pem" + "errors" "slices" "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" + certsv1beta1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/utils/ptr" ) -func TestExtKeyUsages(t *testing.T) { +// fixedClock is a PassiveClock frozen at a fixed instant. +type fixedClock struct { + now time.Time +} + +func (c fixedClock) Now() time.Time { return c.now } +func (c fixedClock) Since(t time.Time) time.Duration { return c.now.Sub(t) } + +// testNow is a whole-second instant so times survive the x509 encoding +// round-trip (certificates carry 1s precision) and compare exactly. It must +// stay near wall-clock time because GenerateED25519CA stamps CA validity +// from time.Now(). +var testNow = time.Now().UTC().Truncate(time.Second) + +// makePodAndPCR returns a pod and a matching PodCertificateRequest with no +// key material set; callers fill in StubPKCS10Request. +func makePodAndPCR(namespace, podName, serviceAccount string, maxExpirationSeconds int32) (*corev1.Pod, *certsv1beta1.PodCertificateRequest) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: podName, + UID: types.UID("pod-uid-1"), + }, + } + pcr := &certsv1beta1.PodCertificateRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "req-1", + }, + Spec: certsv1beta1.PodCertificateRequestSpec{ + SignerName: Name, + PodName: pod.ObjectMeta.Name, + PodUID: pod.ObjectMeta.UID, + ServiceAccountName: serviceAccount, + ServiceAccountUID: types.UID("sa-uid-1"), + NodeName: types.NodeName("node-1"), + NodeUID: types.UID("node-uid-1"), + MaxExpirationSeconds: ptr.To(maxExpirationSeconds), + }, + } + return pod, pcr +} + +// stubCSR returns a stub PKCS#10 request carrying priv's public key. +func stubCSR(t *testing.T, priv ed25519.PrivateKey) []byte { + t.Helper() + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, priv) + if err != nil { + t.Fatalf("while creating stub CSR: %v", err) + } + return csr +} + +func TestMakeCert(t *testing.T) { + testCases := []struct { + name string + namespace string + podName string + serviceAccount string + maxExpirationSeconds int32 + wantLifetime time.Duration + wantURI string + wantEKUs []x509.ExtKeyUsage + wantIdentity *substratex509.PodIdentity + }{ + { + name: "atelet in ate-system", + namespace: "ate-system", + podName: "atelet-abcde", + serviceAccount: "atelet", + maxExpirationSeconds: 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/ate-system/sa/atelet", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid-1", + PodName: "atelet-abcde", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "ordinary workload is client-only", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "requested lifetime capped at 24h", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 7 * 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "shorter requested lifetime honored", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 3600, + wantLifetime: time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + subjectPub, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + + pod, pcr := makePodAndPCR(tc.namespace, tc.podName, tc.serviceAccount, tc.maxExpirationSeconds) + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + + kc := fake.NewSimpleClientset(pod, pcr) + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err != nil { + t.Fatalf("MakeCert: %v", err) + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests(tc.namespace).Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching updated PCR: %v", err) + } + if len(gotPCR.Status.Conditions) != 1 || gotPCR.Status.Conditions[0].Type != certsv1beta1.PodCertificateRequestConditionTypeIssued { + t.Fatalf("PCR status not marked Issued: %+v", gotPCR.Status.Conditions) + } + + block, rest := pem.Decode([]byte(gotPCR.Status.CertificateChain)) + if block == nil { + t.Fatalf("certificate chain contains no PEM block") + } + if len(rest) != 0 { + t.Errorf("expected exactly one certificate in chain (no intermediates), got trailing data") + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("while parsing leaf certificate: %v", err) + } + + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: roots, + CurrentTime: testNow, + KeyUsages: tc.wantEKUs, + }); err != nil { + t.Errorf("leaf does not verify against CA root: %v", err) + } + + leafPub, ok := leaf.PublicKey.(ed25519.PublicKey) + if !ok || !leafPub.Equal(subjectPub) { + t.Errorf("leaf public key %v is not the subject key %v", leaf.PublicKey, subjectPub) + } + + wantNotBefore := testNow.Add(-2 * time.Minute) + wantNotAfter := wantNotBefore.Add(tc.wantLifetime) + wantBeginRefreshAt := wantNotAfter.Add(-30 * time.Minute) + if !leaf.NotBefore.Equal(wantNotBefore) { + t.Errorf("got NotBefore %v, want %v", leaf.NotBefore, wantNotBefore) + } + if !leaf.NotAfter.Equal(wantNotAfter) { + t.Errorf("got NotAfter %v, want %v", leaf.NotAfter, wantNotAfter) + } + if gotPCR.Status.NotBefore == nil || !gotPCR.Status.NotBefore.Time.Equal(wantNotBefore) { + t.Errorf("got status NotBefore %v, want %v", gotPCR.Status.NotBefore, wantNotBefore) + } + if gotPCR.Status.NotAfter == nil || !gotPCR.Status.NotAfter.Time.Equal(wantNotAfter) { + t.Errorf("got status NotAfter %v, want %v", gotPCR.Status.NotAfter, wantNotAfter) + } + if gotPCR.Status.BeginRefreshAt == nil || !gotPCR.Status.BeginRefreshAt.Time.Equal(wantBeginRefreshAt) { + t.Errorf("got status BeginRefreshAt %v, want %v", gotPCR.Status.BeginRefreshAt, wantBeginRefreshAt) + } + + if len(leaf.URIs) != 1 || leaf.URIs[0].String() != tc.wantURI { + t.Errorf("got URIs %v, want [%s]", leaf.URIs, tc.wantURI) + } + if !slices.Equal(leaf.ExtKeyUsage, tc.wantEKUs) { + t.Errorf("got EKUs %v, want %v", leaf.ExtKeyUsage, tc.wantEKUs) + } + if !bytes.Equal(leaf.AuthorityKeyId, ca.RootCertificate.SubjectKeyId) { + t.Errorf("got AuthorityKeyId %x, want CA SubjectKeyId %x", leaf.AuthorityKeyId, ca.RootCertificate.SubjectKeyId) + } + + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil { + t.Fatalf("while extracting PodIdentity: %v", err) + } + if *identity != *tc.wantIdentity { + t.Errorf("got PodIdentity %+v, want %+v", identity, tc.wantIdentity) + } + }) + } +} + +func TestMakeCertErrors(t *testing.T) { testCases := []struct { - name string - namespace string - serviceAccount string - want []x509.ExtKeyUsage + name string + omitPod bool + podUID types.UID + omitKey bool + failUpdate bool }{ { - name: "atelet gets clientAuth and serverAuth", - namespace: "ate-system", - serviceAccount: "atelet", - want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + name: "pod not found", + omitPod: true, + podUID: "pod-uid-1", }, { - name: "other SA in ate-system is client-only", - namespace: "ate-system", - serviceAccount: "ate-controller", - want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + name: "pod UID mismatch", + podUID: "other-uid", }, { - name: "atelet SA name in another namespace is client-only", - namespace: "default", - serviceAccount: "atelet", - want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + name: "no key material in PCR", + podUID: "pod-uid-1", + omitKey: true, }, { - name: "ordinary workload is client-only", - namespace: "default", - serviceAccount: "default", - want: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + name: "status update fails", + podUID: "pod-uid-1", + failUpdate: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got := extKeyUsages(tc.namespace, tc.serviceAccount) - if !slices.Equal(got, tc.want) { - t.Errorf("extKeyUsages(%q, %q) = %v, want %v", tc.namespace, tc.serviceAccount, got, tc.want) + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400) + pod.ObjectMeta.UID = tc.podUID + if !tc.omitKey { + _, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + } + + objects := []runtime.Object{pcr} + if !tc.omitPod { + objects = append(objects, pod) + } + kc := fake.NewSimpleClientset(objects...) + if tc.failUpdate { + kc.PrependReactor("update", "podcertificaterequests", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("injected update failure") + }) + } + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err == nil { + t.Fatalf("MakeCert: got nil error, want error") + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests("ate-system").Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching PCR: %v", err) + } + if len(gotPCR.Status.Conditions) != 0 || gotPCR.Status.CertificateChain != "" { + t.Errorf("PCR status updated despite error: %+v", gotPCR.Status) } }) } } + +func TestMakeCertChainIncludesIntermediates(t *testing.T) { + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + intermediateCA, err := localca.GenerateED25519CA("test-intermediate") + if err != nil { + t.Fatalf("while generating intermediate CA: %v", err) + } + ca.IntermediateCertificates = []*x509.Certificate{intermediateCA.RootCertificate} + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + _, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + + pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400) + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + + kc := fake.NewSimpleClientset(pod, pcr) + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err != nil { + t.Fatalf("MakeCert: %v", err) + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests("ate-system").Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching updated PCR: %v", err) + } + + var chainDER [][]byte + rest := []byte(gotPCR.Status.CertificateChain) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + chainDER = append(chainDER, block.Bytes) + } + if len(rest) != 0 { + t.Errorf("certificate chain has trailing non-PEM data") + } + if len(chainDER) != 2 { + t.Fatalf("got %d certificates in chain, want 2 (leaf + intermediate)", len(chainDER)) + } + if _, err := x509.ParseCertificate(chainDER[0]); err != nil { + t.Errorf("while parsing leaf certificate: %v", err) + } + if !bytes.Equal(chainDER[1], intermediateCA.RootCertificate.Raw) { + t.Errorf("second chain entry is not the intermediate certificate") + } +} + +func TestDesiredClusterTrustBundles(t *testing.T) { + ca1, err := localca.GenerateED25519CA("test-ca-1") + if err != nil { + t.Fatalf("while generating CA 1: %v", err) + } + ca2, err := localca.GenerateED25519CA("test-ca-2") + if err != nil { + t.Fatalf("while generating CA 2: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca1, ca2}} + impl := NewImpl(nil, caPool, fixedClock{now: testNow}) + + ctbs := impl.DesiredClusterTrustBundles() + if len(ctbs) != 1 { + t.Fatalf("got %d ClusterTrustBundles, want 1", len(ctbs)) + } + ctb := ctbs[0] + + if want := CTBPrefix + "primary-bundle"; ctb.ObjectMeta.Name != want { + t.Errorf("got CTB name %q, want %q", ctb.ObjectMeta.Name, want) + } + if ctb.Spec.SignerName != Name { + t.Errorf("got signer name %q, want %q", ctb.Spec.SignerName, Name) + } + if got := ctb.ObjectMeta.Labels["podcert.ate.dev/canarying"]; got != "live" { + t.Errorf("got canarying label %q, want %q", got, "live") + } + + wantBundle := &bytes.Buffer{} + for _, ca := range caPool.CAs { + wantBundle.Write(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: ca.RootCertificate.Raw, + })) + } + if ctb.Spec.TrustBundle != wantBundle.String() { + t.Errorf("got trust bundle:\n%s\nwant:\n%s", ctb.Spec.TrustBundle, wantBundle.String()) + } +} diff --git a/internal/substratex509/substratex509.go b/internal/substratex509/substratex509.go new file mode 100644 index 000000000..ee6ee3b40 --- /dev/null +++ b/internal/substratex509/substratex509.go @@ -0,0 +1,131 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package substratex509 contains routines for creating and parsing x509 +// certificates that embed Substrate-specific X.509 extensions communicating +// the identity of a given workload. It is modeled on the upstream Kubernetes +// component-helpers/kubernetesx509 package, but encodes extension values as +// JSON instead of ASN.1. +package substratex509 + +import ( + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "fmt" + "strings" +) + +var ( + // GoogleSubstratePEN is the ASN.1 Private Enterprise Number arc used to + // name X.509 extensions that communicate Substrate-specific concepts. + GoogleSubstratePEN = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 12} + + // oidPodIdentity identifies the Kubernetes PodIdentity X.509 extension specifically in substrate. + oidPodIdentity = makeSubstrateOID(1) +) + +func makeSubstrateOID(subIDs ...int) asn1.ObjectIdentifier { + base := asn1.ObjectIdentifier{} + base = append(base, GoogleSubstratePEN...) + base = append(base, subIDs...) + return base +} + +// PodIdentity is the Kubernetes Pod Identity of a pod, as embedded in the +// oidPodIdentity extension of its certificate. +type PodIdentity struct { + Namespace string + ServiceAccountName string + ServiceAccountUID string + PodName string + PodUID string + NodeName string + NodeUID string +} + +func AddPodIdentityToCertificate(pod *PodIdentity, template *x509.Certificate) error { + if err := validatePodIdentity(pod); err != nil { + return fmt.Errorf("while validating PodIdentity input: %w", err) + } + podIdentityBytes, err := json.Marshal(pod) + if err != nil { + return fmt.Errorf("while json-marshaling PodIdentity extension: %w", err) + } + + template.ExtraExtensions = append(template.ExtraExtensions, pkix.Extension{ + Id: oidPodIdentity, + Value: podIdentityBytes, + }) + + return nil +} + +func PodIdentityFromCertificate(cert *x509.Certificate) (*PodIdentity, error) { + podIdentityCount := 0 + + pod := &PodIdentity{} + for _, ext := range cert.Extensions { + if ext.Id.Equal(oidPodIdentity) { + podIdentityCount++ + + if err := json.Unmarshal(ext.Value, pod); err != nil { + return nil, fmt.Errorf("while json-unmarshaling PodIdentity extension: %w", err) + } + } + } + + if podIdentityCount == 0 { + return nil, nil + } + if podIdentityCount > 1 { + return nil, fmt.Errorf("certificate contains multiple PodIdentity extensions") + } + + if err := validatePodIdentity(pod); err != nil { + return nil, fmt.Errorf("while validating PodIdentity extension: %w", err) + } + + return pod, nil +} + +func validatePodIdentity(pod *PodIdentity) error { + var empty []string + if pod.Namespace == "" { + empty = append(empty, "Namespace") + } + if pod.ServiceAccountName == "" { + empty = append(empty, "ServiceAccountName") + } + if pod.ServiceAccountUID == "" { + empty = append(empty, "ServiceAccountUID") + } + if pod.PodName == "" { + empty = append(empty, "PodName") + } + if pod.PodUID == "" { + empty = append(empty, "PodUID") + } + if pod.NodeName == "" { + empty = append(empty, "NodeName") + } + if pod.NodeUID == "" { + empty = append(empty, "NodeUID") + } + if len(empty) > 0 { + return fmt.Errorf("empty fields: %s", strings.Join(empty, ", ")) + } + return nil +} diff --git a/internal/substratex509/substratex509_test.go b/internal/substratex509/substratex509_test.go new file mode 100644 index 000000000..42aae802e --- /dev/null +++ b/internal/substratex509/substratex509_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package substratex509 + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "math/big" + "reflect" + "strings" + "testing" + "time" +) + +// mintCert self-signs a certificate from the template and parses it back, so +// that ExtraExtensions round-trip into Extensions. +func mintCert(t *testing.T, template *x509.Certificate) *x509.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + template.SerialNumber = big.NewInt(1) + template.NotBefore = time.Now().Add(-time.Hour) + template.NotAfter = time.Now().Add(time.Hour) + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing certificate: %v", err) + } + return cert +} + +// mintCertWithExtension mints a certificate carrying a single PodIdentity +// extension with the given raw value, bypassing the validation in +// AddPodIdentityToCertificate. +func mintCertWithExtension(t *testing.T, value []byte) *x509.Certificate { + t.Helper() + return mintCert(t, &x509.Certificate{ + ExtraExtensions: []pkix.Extension{ + {Id: testPodIdentityOID, Value: value}, + }, + }) +} + +// testPodIdentityOID spells out the intended PodIdentity extension OID +// to prevent accidental modification to the value. +var testPodIdentityOID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 12, 1} + +func TestPodIdentityFromCertificate(t *testing.T) { + // fullPodIdentity has every field populated, as required by + // AddPodIdentityToCertificate. Cases that mutate it copy it first. + fullPodIdentity := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + for _, tc := range []struct { + name string + cert func(t *testing.T) *x509.Certificate + want *PodIdentity + wantErr string // substring of the expected error; "" means no error + }{ + { + name: "RoundTrip", + cert: func(t *testing.T) *x509.Certificate { + template := &x509.Certificate{} + if err := AddPodIdentityToCertificate(&fullPodIdentity, template); err != nil { + t.Fatalf("AddPodIdentityToCertificate: %v", err) + } + return mintCert(t, template) + }, + want: &fullPodIdentity, + }, + { + name: "Absent", + cert: func(t *testing.T) *x509.Certificate { + return mintCert(t, &x509.Certificate{}) + }, + want: nil, + }, + { + name: "Duplicate", + cert: func(t *testing.T) *x509.Certificate { + // Go's x509 parser rejects certificates with duplicate + // extensions, so a duplicate can only reach + // PodIdentityFromCertificate via a Certificate constructed + // by other means. Build one directly. + ext := pkix.Extension{Id: testPodIdentityOID, Value: []byte(`{"PodUID":"pod-uid"}`)} + return &x509.Certificate{Extensions: []pkix.Extension{ext, ext}} + }, + wantErr: "multiple PodIdentity extensions", + }, + { + name: "EmptyField", + cert: func(t *testing.T) *x509.Certificate { + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + NodeName: "node-1", + NodeUID: "node-uid", + } + value, err := json.Marshal(pod) + if err != nil { + t.Fatalf("marshaling PodIdentity: %v", err) + } + return mintCertWithExtension(t, value) + }, + wantErr: "PodUID", + }, + { + name: "Malformed", + cert: func(t *testing.T) *x509.Certificate { + return mintCertWithExtension(t, []byte("not json")) + }, + wantErr: "json-unmarshaling", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := PodIdentityFromCertificate(tc.cert(t)) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("PodIdentityFromCertificate succeeded, want error containing %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("PodIdentityFromCertificate error = %q, want it to contain %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("PodIdentityFromCertificate: %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("PodIdentityFromCertificate = %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestAddPodIdentityToCertificateEmptyField(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*PodIdentity) + }{ + {"Namespace", func(p *PodIdentity) { p.Namespace = "" }}, + {"ServiceAccountName", func(p *PodIdentity) { p.ServiceAccountName = "" }}, + {"ServiceAccountUID", func(p *PodIdentity) { p.ServiceAccountUID = "" }}, + {"PodName", func(p *PodIdentity) { p.PodName = "" }}, + {"PodUID", func(p *PodIdentity) { p.PodUID = "" }}, + {"NodeName", func(p *PodIdentity) { p.NodeName = "" }}, + {"NodeUID", func(p *PodIdentity) { p.NodeUID = "" }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + tc.mutate(&pod) + err := AddPodIdentityToCertificate(&pod, &x509.Certificate{}) + if err == nil { + t.Fatalf("AddPodIdentityToCertificate succeeded, want error for empty %s", tc.name) + } + if !strings.Contains(err.Error(), tc.name) { + t.Errorf("error %q does not name the empty field %s", err, tc.name) + } + }) + } +} + +// The extension value is plain JSON so that non-Go verifiers can parse it +// without an ASN.1 library. +func TestExtensionValueIsJSON(t *testing.T) { + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + template := &x509.Certificate{} + if err := AddPodIdentityToCertificate(&pod, template); err != nil { + t.Fatalf("AddPodIdentityToCertificate: %v", err) + } + cert := mintCert(t, template) + + for _, ext := range cert.Extensions { + if ext.Id.Equal(testPodIdentityOID) { + if !json.Valid(ext.Value) { + t.Errorf("extension value is not valid JSON: %q", ext.Value) + } + return + } + } + t.Fatal("PodIdentity extension not found in certificate") +} From d32f7a7a8a890ab9362107ecb146931a538f3ddc Mon Sep 17 00:00:00 2001 From: zoezhao Date: Sat, 18 Jul 2026 12:15:02 -0700 Subject: [PATCH 9/9] deduplicated workerpool-ca-certs and atelet-server-trust-bundle, since they both use podidentity signer CA pool --- cmd/ateapi/main.go | 27 ++++++++++------------- cmd/kubectl-ate/README.md | 4 ++-- manifests/ate-install/ate-api-server.yaml | 15 ++----------- 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 072bcaf8e..ec690e780 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -68,11 +68,9 @@ var ( clientJWTAudience = pflag.String("client-jwt-audience", "", "The expected audience for client JWTs.") sessionIDJWTPoolFile = pflag.String("session-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing session JWTs") - sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") - workerpoolCACerts = pflag.String("workerpool-ca-certs", "", "The file that contains the CA for verifying workerpool client certificates.") - - ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle presented as the client certificate when dialing atelet.") - ateletCACerts = pflag.String("atelet-ca-certs", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify atelet serving certificates.") + sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") + podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") + ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") showVersion = pflag.Bool("version", false, "Print version and exit.") clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.") @@ -148,12 +146,12 @@ func main() { ateletPodInformerFactory.WaitForCacheSync(stopCh) ateFactory.WaitForCacheSync(stopCh) - ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *ateletCACerts) + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, ateletDialer, clientset) jwtIssuerDiscoveryClient := buildK8sServiceAccountIssuerDiscoveryClient(ctx, *clientJWTCAFile, *clientJWTIssuer) - sessionIdentitySrv := sessionidentity.New(*clientJWTIssuer, *clientJWTAudience, *sessionIDJWTPoolFile, *sessionIDCAPoolFile, *workerpoolCACerts, jwtIssuerDiscoveryClient) + sessionIdentitySrv := sessionidentity.New(*clientJWTIssuer, *clientJWTAudience, *sessionIDJWTPoolFile, *sessionIDCAPoolFile, *podIdentityCACerts, jwtIssuerDiscoveryClient) debugSrv := debugapi.NewService(redisPersistence) lisCfg := &net.ListenConfig{} @@ -236,9 +234,8 @@ func logFlagValues(ctx context.Context) { slog.String("client-jwt-audience", *clientJWTAudience), slog.String("session-id-jwt-pool", *sessionIDJWTPoolFile), slog.String("session-id-ca-pool", *sessionIDCAPoolFile), - slog.String("workerpool-ca-certs", *workerpoolCACerts), + slog.String("pod-identity-ca-certs", *podIdentityCACerts), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), - slog.String("atelet-ca-certs", *ateletCACerts), ) } @@ -345,22 +342,22 @@ func newKubeClients() (*kubernetes.Clientset, versioned.Interface, error) { return clientset, ateClient, nil } -// buildServerCreds loads the workerpool CA pool (if configured) and +// buildServerCreds loads the pod-identity CA pool (if configured) and // composes gRPC TransportCredentials over the server bundle + optional // client-cert verification. func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, error) { var clientCAs *x509.CertPool - if *workerpoolCACerts != "" { + if *podIdentityCACerts != "" { // TODO: Periodically reload these to handle rotations. Consult with Tina to see how she did it for client-go. - ca, err := os.ReadFile(*workerpoolCACerts) + ca, err := os.ReadFile(*podIdentityCACerts) if err != nil { - return nil, fmt.Errorf("read workerpool CA: %w", err) + return nil, fmt.Errorf("read pod-identity CA: %w", err) } clientCAs = x509.NewCertPool() if !clientCAs.AppendCertsFromPEM(ca) { - return nil, fmt.Errorf("parse workerpool CA from %s", *workerpoolCACerts) + return nil, fmt.Errorf("parse pod-identity CA from %s", *podIdentityCACerts) } - slog.InfoContext(ctx, "Using custom CA for workerpool clients", slog.String("path", *workerpoolCACerts)) + slog.InfoContext(ctx, "Using pod-identity CA for client-cert verification", slog.String("path", *podIdentityCACerts)) } return credentials.NewTLS(&tls.Config{ GetCertificate: credbundle.Loader(*grpcServerCredBundle), diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 54712e826..61f75324a 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -180,9 +180,9 @@ Logs are streamable only while the actor is bound to a worker (i.e., `STATUS_RUN Commands for bootstrapping the Substrate control plane and debugging local environments. ```bash -# Generate a new CA pool and push it directly to a Kubernetes Secret +# Generate a new Session ID CA pool and push it directly to a Kubernetes Secret kubectl ate admin make-ca-pool \ - --name workerpool-ca-certs \ + --name session-id-ca-pool \ --secret-namespace ate-system \ --ca-id "1" diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 0facdd519..5322a661a 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -90,7 +90,8 @@ spec: - --client-jwt-audience=api.ate-system.svc - --session-id-jwt-pool=/run/session-id-jwt-pool/pool.json - --session-id-ca-pool=/run/session-id-ca-pool/pool.json - - --workerpool-ca-certs=/run/workerpool-ca-certs/trust-bundle.pem + - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem env: - name: POD_NAME valueFrom: @@ -134,9 +135,6 @@ spec: - name: "session-id-ca-pool" mountPath: "/run/session-id-ca-pool" readOnly: true - - name: "workerpool-ca-certs" - mountPath: "/run/workerpool-ca-certs" - readOnly: true ports: - containerPort: 443 - name: prometheus @@ -192,15 +190,6 @@ spec: items: - key: "pool" path: "pool.json" - - name: "workerpool-ca-certs" - projected: - sources: - - clusterTrustBundle: - signerName: podidentity.podcert.ate.dev/identity - labelSelector: - matchLabels: - podcert.ate.dev/canarying: live - path: trust-bundle.pem --- # 6. Expose the Session Assigner apiVersion: v1