diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 5a6c99eec..3da77e517 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -43,8 +43,8 @@ jobs: strategy: fail-fast: false matrix: - auth-mode: [mtls, jwt] - name: e2e-test (${{ matrix.auth-mode }}) + ateapi-client-auth: [cert, token] + name: e2e-test (${{ matrix.ateapi-client-auth }}) steps: - name: Checkout uses: actions/checkout@v5 @@ -88,11 +88,11 @@ jobs: - name: Create cluster run: hack/create-kind-cluster.sh - name: Install Agent Substrate - run: hack/install-ate-kind.sh --deploy-ate-system --auth-mode=${{ matrix.auth-mode }} + run: hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and applies the # counter-microvm demo onto the control plane installed above. - run: hack/run-microvm-demo-kind.sh --auth-mode=${{ matrix.auth-mode }} + run: hack/run-microvm-demo-kind.sh --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Wait for micro-VM golden snapshot diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 89e8e37a6..3cc4fad9f 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -15,12 +15,18 @@ package controlapi import ( + "crypto/tls" + "crypto/x509" "errors" "fmt" + "os" + + "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/substratex509" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -33,14 +39,26 @@ type AteletDialer struct { workerIndexer cache.Indexer ateletIndexer cache.Indexer ateletConns *lru.Cache + // dialCredentials builds the transport credentials used to dial a given + // atelet, keyed on the atelet's expected pod UID. Production wires this to + // per-atelet mTLS; tests can override it with insecure credentials. + dialCredentials func(expectedPodUID string) (credentials.TransportCredentials, error) } -// NewAteletDialer creates a new AteletDialer. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer) *AteletDialer { +// NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath +// are used to build the per-atelet mTLS credentials used for every atelet connection. +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string) *AteletDialer { return &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: lru.New(1024), + dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) { + tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID) + if err != nil { + return nil, err + } + return credentials.NewTLS(tlsConfig), nil + }, } } @@ -73,7 +91,7 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( } selectedAtelet := matchingAtelets[0].(*corev1.Pod) - ateletKey := selectedAtelet.ObjectMeta.Namespace + "/" + selectedAtelet.ObjectMeta.Name + ateletKey := string(selectedAtelet.ObjectMeta.UID) ateletConnAny, ok := d.ateletConns.Get(ateletKey) if ok { @@ -81,12 +99,17 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( } if len(selectedAtelet.Status.PodIPs) == 0 { - return nil, fmt.Errorf("selected atelet %q has no assigned IPs: %w", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name, err) + return nil, fmt.Errorf("selected atelet %q has no assigned IPs", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name) + } + + creds, err := d.dialCredentials(string(selectedAtelet.ObjectMeta.UID)) + if err != nil { + return nil, fmt.Errorf("while building atelet credentials: %w", err) } ateletConn, err := grpc.NewClient( selectedAtelet.Status.PodIPs[0].IP+":8085", - grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithTransportCredentials(creds), grpc.WithStatsHandler(otelgrpc.NewClientHandler()), ) if err != nil { @@ -97,3 +120,70 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) ( return ateletConn, nil } + +func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) { + roots, err := caPoolFromFile(serverCAPath) + if err != nil { + return nil, err + } + + tlsConfig := tls.Config{ + MinVersion: tls.VersionTLS13, + GetClientCertificate: credbundle.ClientLoader(clientBundlePath), + // Skip the default verification because the peer is dialed by IP and its + // certificate has no DNS/IP SAN. + InsecureSkipVerify: true, + RootCAs: roots, + VerifyConnection: verifyAteletServerCert(roots, expectedPodUID), + } + + return &tlsConfig, nil +} + +func verifyAteletServerCert(roots *x509.CertPool, expectedPodUID string) func(tls.ConnectionState) error { + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + return fmt.Errorf("server presented no certificate") + } + leaf := cs.PeerCertificates[0] + intermediates := x509.NewCertPool() + for _, c := range cs.PeerCertificates[1:] { + intermediates.AddCert(c) + } + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: roots, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + return fmt.Errorf("verifying server certificate chain: %w", err) + } + + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil { + return fmt.Errorf("failed to parse PodIdentity extension: %w", err) + } + if identity == nil { + return fmt.Errorf("server certificate has no PodIdentity extension, expected pod UID %q", expectedPodUID) + } + if identity.PodUID == "" || expectedPodUID == "" { + return fmt.Errorf("found pod UID == %q, expected pod UID == %q", identity.PodUID, expectedPodUID) + } + if identity.PodUID != expectedPodUID { + return fmt.Errorf("pod UID %q does not match expected %q", identity.PodUID, expectedPodUID) + } + + return nil + } +} + +func caPoolFromFile(path string) (*x509.CertPool, error) { + caBytes, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read CA bundle from %s: %w", path, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caBytes) { + return nil, fmt.Errorf("failed to parse CA bundle from %s", path) + } + return pool, nil +} diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go new file mode 100644 index 000000000..f113f9db0 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -0,0 +1,160 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/substratex509" +) + +// makeTestCA mints a self-signed CA and returns it along with a pool +// containing it as the sole root. +func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509.CertPool) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating CA key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing CA certificate: %v", err) + } + pool := x509.NewCertPool() + pool.AddCert(cert) + return cert, key, pool +} + +// makeLeafCert mints a server leaf certificate signed by the given CA. If +// podUID is non-empty it is embedded in a PodIdentity extension. +func makeLeafCert(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, podUID string) *x509.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating leaf key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if podUID != "" { + // AddPodIdentityToCertificate requires all fields to be non-empty; + // only PodUID matters to these tests. + err := substratex509.AddPodIdentityToCertificate(&substratex509.PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: podUID, + NodeName: "node-1", + NodeUID: "node-uid", + }, template) + if err != nil { + t.Fatalf("adding PodIdentity extension: %v", err) + } + } + der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey) + if err != nil { + t.Fatalf("creating leaf certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing leaf certificate: %v", err) + } + return cert +} + +func TestVerifyAteletServerCert(t *testing.T) { + ca, caKey, roots := makeTestCA(t) + otherCA, otherCAKey, _ := makeTestCA(t) + + const uid = "5a2e1c9f-0b57-4a52-9f6e-2f6d3a1b8c4d" + + tests := []struct { + name string + leaf *x509.Certificate + expectedUID string + wantErr bool + }{ + { + name: "matching UID succeeds", + leaf: makeLeafCert(t, ca, caKey, uid), + expectedUID: uid, + }, + { + name: "mismatched UID fails", + leaf: makeLeafCert(t, ca, caKey, "some-other-uid"), + expectedUID: uid, + wantErr: true, + }, + { + name: "missing pod UID extension fails", + leaf: makeLeafCert(t, ca, caKey, ""), + expectedUID: uid, + wantErr: true, + }, + { + name: "cert from untrusted CA fails", + leaf: makeLeafCert(t, otherCA, otherCAKey, uid), + expectedUID: uid, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + verify := verifyAteletServerCert(roots, tc.expectedUID) + err := verify(tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{tc.leaf}, + }) + if gotErr := err != nil; gotErr != tc.wantErr { + t.Fatalf("verify returned error %v, wantErr=%v", err, tc.wantErr) + } + }) + } + + t.Run("no peer certificate fails", func(t *testing.T) { + verify := verifyAteletServerCert(roots, uid) + if err := verify(tls.ConnectionState{}); err == nil { + t.Fatal("verify succeeded, want error") + } + }) +} diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index 51ac51022..58f5a7bc4 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -42,6 +42,7 @@ import ( "github.com/redis/go-redis/v9" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -309,7 +310,13 @@ func setupTest(t *testing.T, ns string) *testContext { t.Fatalf("failed to start worker cache: %v", err) } - dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer()) + dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "") + // Dial the fake atelet over insecure transport instead of per-atelet mTLS, + // so DialForWorker's real lookup/dial/cache path is exercised under test. + dialer.dialCredentials = func(_ string) (credentials.TransportCredentials, error) { + return insecure.NewCredentials(), nil + } + service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient) // 5. Start REAL gRPC Server for ATE API diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index c08feb51c..ec690e780 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -28,7 +28,6 @@ import ( "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/credbundle" "github.com/agent-substrate/substrate/cmd/ateapi/internal/debugapi" "github.com/agent-substrate/substrate/cmd/ateapi/internal/k8sjwt" "github.com/agent-substrate/substrate/cmd/ateapi/internal/sessionidentity" @@ -36,6 +35,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" @@ -68,11 +68,11 @@ var ( clientJWTAudience = pflag.String("client-jwt-audience", "", "The expected audience for client JWTs.") sessionIDJWTPoolFile = pflag.String("session-id-jwt-pool", "", "The file that contains the serialized JWT authority pool for signing session JWTs") - sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") - workerpoolCACerts = pflag.String("workerpool-ca-certs", "", "The file that contains the CA for verifying workerpool client certificates.") + sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") + podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") + ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") showVersion = pflag.Bool("version", false, "Print version and exit.") - authMode = pflag.String("auth-mode", "mtls", "Auth mode for incoming gRPC: mtls|jwt. 'mtls' (default) relies on transport-level mTLS for client identity. 'jwt' additionally requires a Kubernetes ServiceAccount Bearer token on every RPC. Substrate will drop support for JWT auth mode once the Pod Certificates feature is enabled by default in the minimum supported Kubernetes version.") clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.") ) @@ -103,11 +103,6 @@ func main() { loadFlagsFromEnv() logFlagValues(ctx) - authModeParsed, err := ateapiauth.ParseMode(*authMode) - if err != nil { - serverboot.Fatal(ctx, "Invalid --auth-mode", err) - } - redisClient, err := connectRedis(ctx) if err != nil { serverboot.Fatal(ctx, "Failed to set up Redis/Valkey", err) @@ -151,15 +146,12 @@ func main() { ateletPodInformerFactory.WaitForCacheSync(stopCh) ateFactory.WaitForCacheSync(stopCh) - dialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer()) - sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, clientset) + ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) + sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, ateletDialer, clientset) jwtIssuerDiscoveryClient := buildK8sServiceAccountIssuerDiscoveryClient(ctx, *clientJWTCAFile, *clientJWTIssuer) - if authModeParsed == ateapiauth.ModeJWT && jwtIssuerDiscoveryClient == nil { - serverboot.Fatal(ctx, "JWT auth mode requires a Kubernetes ServiceAccount issuer discovery client", fmt.Errorf("client JWT issuer %q is not usable for discovery", *clientJWTIssuer)) - } - sessionIdentitySrv := sessionidentity.New(*clientJWTIssuer, *clientJWTAudience, *sessionIDJWTPoolFile, *sessionIDCAPoolFile, *workerpoolCACerts, jwtIssuerDiscoveryClient) + sessionIdentitySrv := sessionidentity.New(*clientJWTIssuer, *clientJWTAudience, *sessionIDJWTPoolFile, *sessionIDCAPoolFile, *podIdentityCACerts, jwtIssuerDiscoveryClient) debugSrv := debugapi.NewService(redisPersistence) lisCfg := &net.ListenConfig{} @@ -169,10 +161,12 @@ func main() { } authCfg := ateapiauth.ServerConfig{ - Mode: authModeParsed, - VerifyBearerToken: func(ctx context.Context, bearer string) error { - _, err := k8sjwt.Verify(ctx, jwtIssuerDiscoveryClient, bearer, *clientJWTIssuer, *clientJWTAudience, time.Now()) - return err + VerifyBearerToken: func(ctx context.Context, bearer string) (string, error) { + claims, err := k8sjwt.Verify(ctx, jwtIssuerDiscoveryClient, bearer, *clientJWTIssuer, *clientJWTAudience, time.Now()) + if err != nil { + return "", err + } + return claims.Subject, nil }, } if err := ateapiauth.ValidateServerConfig(authCfg); err != nil { @@ -240,8 +234,8 @@ func logFlagValues(ctx context.Context) { slog.String("client-jwt-audience", *clientJWTAudience), slog.String("session-id-jwt-pool", *sessionIDJWTPoolFile), slog.String("session-id-ca-pool", *sessionIDCAPoolFile), - slog.String("workerpool-ca-certs", *workerpoolCACerts), - slog.String("auth-mode", *authMode), + slog.String("pod-identity-ca-certs", *podIdentityCACerts), + slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), ) } @@ -285,7 +279,7 @@ func connectRedis(ctx context.Context) (*redis.ClusterClient, error) { } func buildRedisTLSConfig(ctx context.Context) (*tls.Config, error) { - tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS13} if *redisCACerts != "" { ca, err := os.ReadFile(*redisCACerts) if err != nil { @@ -348,27 +342,30 @@ func newKubeClients() (*kubernetes.Clientset, versioned.Interface, error) { return clientset, ateClient, nil } -// buildServerCreds loads the workerpool CA pool (if configured) and +// buildServerCreds loads the pod-identity CA pool (if configured) and // composes gRPC TransportCredentials over the server bundle + optional // client-cert verification. func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, error) { var clientCAs *x509.CertPool - if *workerpoolCACerts != "" { + if *podIdentityCACerts != "" { // TODO: Periodically reload these to handle rotations. Consult with Tina to see how she did it for client-go. - ca, err := os.ReadFile(*workerpoolCACerts) + ca, err := os.ReadFile(*podIdentityCACerts) if err != nil { - return nil, fmt.Errorf("read workerpool CA: %w", err) + return nil, fmt.Errorf("read pod-identity CA: %w", err) } clientCAs = x509.NewCertPool() if !clientCAs.AppendCertsFromPEM(ca) { - return nil, fmt.Errorf("parse workerpool CA from %s", *workerpoolCACerts) + return nil, fmt.Errorf("parse pod-identity CA from %s", *podIdentityCACerts) } - slog.InfoContext(ctx, "Using custom CA for workerpool clients", slog.String("path", *workerpoolCACerts)) + slog.InfoContext(ctx, "Using pod-identity CA for client-cert verification", slog.String("path", *podIdentityCACerts)) } return credentials.NewTLS(&tls.Config{ GetCertificate: credbundle.Loader(*grpcServerCredBundle), - ClientAuth: tls.VerifyClientCertIfGiven, - ClientCAs: clientCAs, + // Client certs stay optional at the transport level: certless + // clients such as kubectl-ate authenticate with a Bearer token in the + // ateapiauth interceptor. + ClientAuth: tls.VerifyClientCertIfGiven, + ClientCAs: clientCAs, }), nil } diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index e96bdb242..fc9edc098 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -39,10 +39,11 @@ var ( ateAPIConnSpec = pflag.String("ateapi-conn-spec", "dns:///api.ate-system.svc:443", "") - ateapiAuthMode = pflag.String("ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.") - ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") - ateapiTokenFile = pflag.String("ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") + ateapiTokenAuth = pflag.Bool("ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") + ateapiTokenFile = pflag.String("ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") + ateapiClientCert = pflag.String("ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required unless --ateapi-use-token-auth is set, ignored otherwise.") ) func init() { @@ -54,17 +55,12 @@ func main() { pflag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) - mode, err := ateapiauth.ParseMode(*ateapiAuthMode) - if err != nil { - setupLog.Error(err, "invalid --ateapi-auth") - os.Exit(1) - } - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: mode, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - TokenFile: *ateapiTokenFile, + UseTokenAuth: *ateapiTokenAuth, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + TokenFile: *ateapiTokenFile, + ClientCredBundle: *ateapiClientCert, }) if err != nil { setupLog.Error(err, "building ateapi dial options") diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index e99e42815..30cc621c2 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -16,6 +16,8 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -34,6 +36,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateerrors" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/credbundle" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -53,6 +56,7 @@ import ( "google.golang.org/api/option" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/reflection" "google.golang.org/grpc/status" @@ -65,6 +69,9 @@ var ( port = pflag.Int("port", 8085, "The port to listen on") metricsListenAddr = pflag.String("metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") + grpcServerCredBundle = pflag.String("grpc-server-cred-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Credential bundle atelet presents as its gRPC serving certificate.") + clientCACerts = pflag.String("client-ca-certs", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify gRPC client certificates.") + gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -173,7 +180,16 @@ func main() { serverboot.Fatal(ctx, "Failed to listen", err) } - svr := grpc.NewServer(grpc.StatsHandler(otelgrpc.NewServerHandler()), grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor)) + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) + } + + svr := grpc.NewServer( + grpc.Creds(credentials.NewTLS(tlsCfg)), + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), + ) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) @@ -1070,3 +1086,23 @@ func resetActorDirs(actorUID string) error { return nil } + +// ateletServerTLSConfig builds a *tls.Config for a gRPC server that presents the +// credential bundle at servingBundlePath, requires a client certificate +// chaining to a CA in clientCAPath. +func ateletServerTLSConfig(servingBundlePath, clientCAPath string) (*tls.Config, error) { + caBytes, err := os.ReadFile(clientCAPath) + if err != nil { + return nil, fmt.Errorf("read CA bundle %s: %w", clientCAPath, err) + } + clientCAs := x509.NewCertPool() + if !clientCAs.AppendCertsFromPEM(caBytes) { + return nil, fmt.Errorf("parse CA bundle from %s", clientCAPath) + } + return &tls.Config{ + MinVersion: tls.VersionTLS13, + GetCertificate: credbundle.Loader(servingBundlePath), + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: clientCAs, + }, nil +} diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index 7c7e66806..53ae85d13 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -18,6 +18,7 @@ import ( "fmt" "os" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router" "github.com/agent-substrate/substrate/internal/version" "github.com/spf13/cobra" ) @@ -37,6 +38,6 @@ func Execute() { } func init() { - rootCmd.AddCommand(NewRouterCmd()) + rootCmd.AddCommand(router.NewRouterCmd()) rootCmd.AddCommand(NewDnsCmd()) } diff --git a/cmd/atenet/internal/router.go b/cmd/atenet/internal/router/cmd.go similarity index 74% rename from cmd/atenet/internal/router.go rename to cmd/atenet/internal/router/cmd.go index c58f31a7e..3cddd048d 100644 --- a/cmd/atenet/internal/router.go +++ b/cmd/atenet/internal/router/cmd.go @@ -12,26 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package router import ( "fmt" "time" "github.com/spf13/cobra" - - "github.com/agent-substrate/substrate/cmd/atenet/internal/router" - "github.com/agent-substrate/substrate/internal/ateapiauth" ) func NewRouterCmd() *cobra.Command { - var cfg router.RouterConfig + var cfg routerConfig cmd := &cobra.Command{ Use: "router", Short: "Router components including xDS server and Envoy ExtProc gateway processing server", RunE: func(cmd *cobra.Command, args []string) error { - srv, err := router.NewRouterServer(cfg) + srv, err := NewRouterServer(cfg) if err != nil { return fmt.Errorf("failed to create router server: %w", err) } @@ -56,12 +53,13 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)") cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services") cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") - cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)") + cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)") - cmd.Flags().StringVar(&cfg.AteapiAuthMode, "ateapi-auth", "mtls", "Client auth to ateapi: mtls|jwt. 'mtls' (default) dials with insecure TLS and relies on pod-projected mTLS credentials for identity. 'jwt' verifies the server cert and sends a Bearer SA token.") - cmd.Flags().StringVar(&cfg.AteapiCAFile, "ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert. Required for jwt.") - cmd.Flags().StringVar(&cfg.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") - cmd.Flags().StringVar(&cfg.AteapiTokenFile, "ateapi-token-file", ateapiauth.DefaultServiceAccountTokenFile, "Projected SA token file used as Bearer credential. Required for jwt.") + cmd.Flags().StringVar(&cfg.Auth.AteapiCAFile, "ateapi-ca-file", "", "PEM file with CAs trusted to verify the ateapi server cert. Required.") + cmd.Flags().StringVar(&cfg.Auth.AteapiClientCertPath, "ateapi-client-cert", "", "Credential bundle presented as the client certificate when dialing ateapi. Required unless --ateapi-use-token-auth is set, ignored otherwise.") + cmd.Flags().StringVar(&cfg.Auth.AteapiServerName, "ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") + cmd.Flags().BoolVar(&cfg.Auth.AteapiUseTokenAuth, "ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") + cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") return cmd } diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go new file mode 100644 index 000000000..83893ea9b --- /dev/null +++ b/cmd/atenet/internal/router/config.go @@ -0,0 +1,58 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "time" +) + +// authConfig holds the router's client-auth settings for dialing ateapi. +// AteapiCAFile always verifies ateapi's serving cert (the servicedns trust +// bundle in-cluster). By default the router presents AteapiClientCertPath +// (the podidentity credential bundle) as its client cert; with +// AteapiUseTokenAuth it sends a Bearer token from AteapiTokenFile instead and +// the cert path is ignored. +type authConfig struct { + AteapiUseTokenAuth bool + AteapiCAFile string + AteapiClientCertPath string + AteapiServerName string + AteapiTokenFile string +} + +// routerConfig holds deployment setup and endpoint options for the router node instance. +type routerConfig struct { + Standalone bool + Namespace string + Kubeconfig string + AteapiAddr string + HttpPort int + XdsPort int + ExtprocPort int + ExtprocAddr string + EnvoyImage string + TemplatesFile string + StatusPort int + HealthInterval time.Duration + HttpsPort int + EnvoyCertPath string + LogLevel string + MetricsAddr string + // OtlpCollectorAddress is the host:port of the OTLP gRPC collector that + // Envoy reports tracing spans to. Empty disables Envoy-side tracing. + OtlpCollectorAddress string + + Auth authConfig +} diff --git a/cmd/atenet/internal/router/controller.go b/cmd/atenet/internal/router/controller.go index 1b0a1ed18..e66243d5b 100644 --- a/cmd/atenet/internal/router/controller.go +++ b/cmd/atenet/internal/router/controller.go @@ -28,7 +28,7 @@ import ( type Controller struct { k8sClient client.Client clientset kubernetes.Interface - cfg RouterConfig + cfg routerConfig xdsSrv *XdsServer extprocSrv *ExtProcServer @@ -39,7 +39,7 @@ type Controller struct { func NewController( k8sClient client.Client, clientset kubernetes.Interface, - cfg RouterConfig, + cfg routerConfig, xdsSrv *XdsServer, extprocSrv *ExtProcServer, ) *Controller { diff --git a/cmd/atenet/internal/router/envoyrunner.go b/cmd/atenet/internal/router/envoyrunner.go index df8765c62..4eee29303 100644 --- a/cmd/atenet/internal/router/envoyrunner.go +++ b/cmd/atenet/internal/router/envoyrunner.go @@ -37,10 +37,10 @@ const ( // Envoy proxy instance running inside Kubernetes. type envoyrunner struct { k8sClient client.Client - cfg RouterConfig + cfg routerConfig } -func newEnvoyRunner(k8sClient client.Client, cfg RouterConfig) *envoyrunner { +func newEnvoyRunner(k8sClient client.Client, cfg routerConfig) *envoyrunner { return &envoyrunner{ k8sClient: k8sClient, cfg: cfg, diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index d9e1fddab..022b47ab1 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -54,11 +54,11 @@ type routerHealth struct { interval time.Duration clientset kubernetes.Interface apiClient ateapipb.ControlClient - cfg RouterConfig + cfg routerConfig envoyClient *http.Client } -func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg RouterConfig) *routerHealth { +func newRouterHealth(interval time.Duration, clientset kubernetes.Interface, apiClient ateapipb.ControlClient, cfg routerConfig) *routerHealth { if interval <= 0 { interval = time.Second } diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 7ed77df73..61cfb0cf4 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -16,22 +16,15 @@ package router import ( "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "errors" "fmt" "log/slog" - "math/big" "net" "net/http" "os" "os/signal" "strings" "syscall" - "time" "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -62,37 +55,9 @@ func init() { utilruntime.Must(v1alpha1.AddToScheme(scheme)) } -// RouterConfig holds deployment setup and endpoint options for the router node instance. -type RouterConfig struct { - Standalone bool - Namespace string - Kubeconfig string - AteapiAddr string - HttpPort int - XdsPort int - ExtprocPort int - ExtprocAddr string - EnvoyImage string - TemplatesFile string - StatusPort int - HealthInterval time.Duration - HttpsPort int - EnvoyCertPath string - LogLevel string - MetricsAddr string - // OtlpCollectorAddress is the host:port of the OTLP gRPC collector that - // Envoy reports tracing spans to. Empty disables Envoy-side tracing. - OtlpCollectorAddress string - - AteapiAuthMode string - AteapiCAFile string - AteapiServerName string - AteapiTokenFile string -} - // RouterServer instantiates and coordinates runtime threads executing system modules. type RouterServer struct { - cfg RouterConfig + cfg routerConfig Cmd *cobra.Command k8sClient client.Client @@ -103,7 +68,7 @@ type RouterServer struct { atStore atStore } -func NewRouterServer(cfg RouterConfig) (*RouterServer, error) { +func NewRouterServer(cfg routerConfig) (*RouterServer, error) { var k8sClient client.Client var clientset kubernetes.Interface @@ -193,15 +158,12 @@ func (s *RouterServer) Run(ctx context.Context) error { go serverboot.StartMetricsServer(ctx, serverboot.MetricsServerOptions{Addr: s.cfg.MetricsAddr}) - authMode, err := ateapiauth.ParseMode(s.cfg.AteapiAuthMode) - if err != nil { - return fmt.Errorf("invalid --ateapi-auth: %w", err) - } dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - Mode: authMode, - CAFile: s.cfg.AteapiCAFile, - ServerName: s.cfg.AteapiServerName, - TokenFile: s.cfg.AteapiTokenFile, + UseTokenAuth: s.cfg.Auth.AteapiUseTokenAuth, + CAFile: s.cfg.Auth.AteapiCAFile, + ServerName: s.cfg.Auth.AteapiServerName, + TokenFile: s.cfg.Auth.AteapiTokenFile, + ClientCredBundle: s.cfg.Auth.AteapiClientCertPath, }) if err != nil { return fmt.Errorf("building ateapi dial options: %w", err) @@ -214,7 +176,7 @@ func (s *RouterServer) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to establish grpc channel to ateapi client: %w", err) } - slog.InfoContext(ctx, "Connecting to ateapi", slog.String("address", s.cfg.AteapiAddr), slog.String("auth", string(authMode))) + slog.InfoContext(ctx, "Connecting to ateapi", slog.String("address", s.cfg.AteapiAddr), slog.Bool("use-api-token-auth", s.cfg.Auth.AteapiUseTokenAuth)) s.apiClient = ateapipb.NewControlClient(conn) slog.InfoContext(ctx, "Starting substrate router subsystem", slog.Bool("standalone", s.cfg.Standalone)) @@ -227,17 +189,7 @@ func (s *RouterServer) Run(ctx context.Context) error { return fmt.Errorf("configure OTLP collector: %w", err) } - var certContent, keyContent string - if s.cfg.EnvoyCertPath == "" { - slog.InfoContext(ctx, "No Envoy certificate path provided, generating self-signed certificate for testing") - var err error - certContent, keyContent, err = generateSelfSignedCert() - if err != nil { - return fmt.Errorf("failed to generate self-signed cert: %w", err) - } - } - - xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath, certContent, keyContent) + xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath) if s.extprocSrv == nil { routeDuration, err := newRouteDurationHistogram() if err != nil { @@ -313,39 +265,3 @@ func (s *RouterServer) Run(ctx context.Context) error { return g.Wait() } - -func generateSelfSignedCert() (string, string, error) { - priv, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return "", "", err - } - - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - Organization: []string{"Substrate Local Test"}, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 365), - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - DNSNames: []string{"localhost"}, - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) - if err != nil { - return "", "", err - } - - certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - - privBytes, err := x509.MarshalPKCS8PrivateKey(priv) - if err != nil { - return "", "", err - } - keyPem := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) - - return string(certPem), string(keyPem), nil -} diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index a72011848..116359ce8 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -16,12 +16,20 @@ package router import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" "io" + "math/big" "net" "net/http" "os" + "path/filepath" "strings" "testing" "time" @@ -47,7 +55,10 @@ func TestStatuszEndpoint(t *testing.T) { defer os.Remove(tmpFile.Name()) tmpFile.Close() - cfg := RouterConfig{ + // Run() dials ateapi in mtls mode, which requires real TLS material; generate it. + caPath, clientCertPath := writeTestTLSMaterial(t) + + cfg := routerConfig{ Standalone: true, Namespace: "default", StatusPort: httpPort, @@ -56,7 +67,10 @@ func TestStatuszEndpoint(t *testing.T) { ExtprocPort: 50051, TemplatesFile: tmpFile.Name(), MetricsAddr: "127.0.0.1:0", - AteapiCAFile: "unused-in-mtls-mode", + Auth: authConfig{ + AteapiCAFile: caPath, + AteapiClientCertPath: clientCertPath, + }, } srv, err := NewRouterServer(cfg) @@ -142,3 +156,44 @@ func TestStatuszEndpoint(t *testing.T) { t.Errorf("Target parameters unassigned inside context payload context properties: found %s", dashboard.Queries[0].Target) } } + +// writeTestTLSMaterial generates a self-signed certificate and writes a CA trust +// bundle and a client credential bundle to temp files, returning their paths. +// Run() (mtls mode) requires both to build its ateapi mTLS credentials. +func writeTestTLSMaterial(t *testing.T) (caPath, clientCertPath string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating certificate: %v", err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshaling key: %v", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + + dir := t.TempDir() + caPath = filepath.Join(dir, "ca.pem") + if err := os.WriteFile(caPath, certPEM, 0o600); err != nil { + t.Fatalf("writing CA file: %v", err) + } + clientCertPath = filepath.Join(dir, "client.pem") + if err := os.WriteFile(clientCertPath, append(certPEM, keyPEM...), 0o600); err != nil { + t.Fatalf("writing client cert file: %v", err) + } + return caPath, clientCertPath +} diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e7e1602a9..3d52657af 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -80,10 +80,8 @@ type XdsServer struct { mu sync.Mutex - httpsPort int - certPath string - certContent string - keyContent string + httpsPort int + certPath string otlpHost string otlpPort uint32 @@ -111,13 +109,11 @@ func (x *XdsServer) SetConfig(ingressPort int, extprocPort int, extprocAddr stri x.extprocAddr = extprocAddr } -func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string, certContent string, keyContent string) { +func (x *XdsServer) SetTlsConfig(httpsPort int, certPath string) { x.mu.Lock() defer x.mu.Unlock() x.httpsPort = httpsPort x.certPath = certPath - x.certContent = certContent - x.keyContent = keyContent } // SetOtlpCollector enables Envoy-side tracing pointed at the OTLP gRPC @@ -576,30 +572,15 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { } func (x *XdsServer) buildTlsCertificate() *tlsv3.TlsCertificate { - if x.certPath != "" { - return &tlsv3.TlsCertificate{ - CertificateChain: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{ - Filename: x.certPath, - }, - }, - PrivateKey: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{ - Filename: x.certPath, // Assuming combined file - }, - }, - } - } - return &tlsv3.TlsCertificate{ CertificateChain: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: x.certContent, + Specifier: &corev3.DataSource_Filename{ + Filename: x.certPath, }, }, PrivateKey: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: x.keyContent, + Specifier: &corev3.DataSource_Filename{ + Filename: x.certPath, // Assuming combined file }, }, } diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 92e347648..9a2c2460f 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -141,7 +141,7 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(8085, 50053, "127.0.0.1") - server.SetTlsConfig(8443, "", "dummy-cert", "dummy-key") + server.SetTlsConfig(8443, "") err := server.UpdateSnapshot() if err != nil { diff --git a/cmd/kubectl-ate/README.md b/cmd/kubectl-ate/README.md index 54712e826..61f75324a 100644 --- a/cmd/kubectl-ate/README.md +++ b/cmd/kubectl-ate/README.md @@ -180,9 +180,9 @@ Logs are streamable only while the actor is bound to a worker (i.e., `STATUS_RUN Commands for bootstrapping the Substrate control plane and debugging local environments. ```bash -# Generate a new CA pool and push it directly to a Kubernetes Secret +# Generate a new Session ID CA pool and push it directly to a Kubernetes Secret kubectl ate admin make-ca-pool \ - --name workerpool-ca-certs \ + --name session-id-ca-pool \ --secret-namespace ate-system \ --ca-id "1" diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go index 5b63a15e4..815d38b6d 100644 --- a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner.go @@ -28,6 +28,7 @@ import ( "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/podcertificate" "github.com/agent-substrate/substrate/cmd/podcertcontroller/internal/signercontroller" "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" certsv1beta1 "k8s.io/api/certificates/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -38,6 +39,22 @@ import ( const Name = "podidentity.podcert.ate.dev/identity" const CTBPrefix = "podidentity.podcert.ate.dev:identity:" +// atelet's identity, as installed by manifests/ate-install/atelet.yaml. Pods +// running as atelet serve TLS (e.g. to ate-apiserver), so their certs also +// carry the serverAuth EKU. +const ( + ateletNamespace = "ate-system" + ateletServiceAccount = "atelet" +) + +func extKeyUsages(namespace, serviceAccount string) []x509.ExtKeyUsage { + usages := []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + if namespace == ateletNamespace && serviceAccount == ateletServiceAccount { + usages = append(usages, x509.ExtKeyUsageServerAuth) + } + return usages +} + type Impl struct { kc kubernetes.Interface caPool *localca.Pool @@ -121,16 +138,38 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq Path: path.Join("ns", pcr.ObjectMeta.Namespace, "sa", pcr.Spec.ServiceAccountName), } + parent := h.caPool.CAs[0].RootCertificate + template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, NotAfter: notAfter, URIs: []*url.URL{spiffeURI}, KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + ExtKeyUsage: extKeyUsages(pcr.ObjectMeta.Namespace, pcr.Spec.ServiceAccountName), + // Link the leaf to its issuing CA by key id so verifiers can disambiguate + // a multi-CA trust bundle (e.g. valkey trusts both the servicedns and + // podidentity CAs). + // https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.1 + AuthorityKeyId: parent.SubjectKeyId, + } + + // Fields are sourced from the PCR spec (attested by kube-apiserver) rather + // than the Pod object, which lacks the ServiceAccount and Node UIDs. + podIdentity := &substratex509.PodIdentity{ + Namespace: pcr.ObjectMeta.Namespace, + ServiceAccountName: pcr.Spec.ServiceAccountName, + ServiceAccountUID: string(pcr.Spec.ServiceAccountUID), + PodName: pcr.Spec.PodName, + PodUID: string(pcr.Spec.PodUID), + NodeName: string(pcr.Spec.NodeName), + NodeUID: string(pcr.Spec.NodeUID), + } + if err := substratex509.AddPodIdentityToCertificate(podIdentity, template); err != nil { + return fmt.Errorf("while adding pod identity to certificate: %w", err) } - subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, h.caPool.CAs[0].RootCertificate, subjectPublicKey, h.caPool.CAs[0].SigningKey) + subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) if err != nil { return fmt.Errorf("while signing subject cert: %w", err) } diff --git a/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go new file mode 100644 index 000000000..6afd1d50c --- /dev/null +++ b/cmd/podcertcontroller/internal/podidentitysigner/podidentitysigner_test.go @@ -0,0 +1,452 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package podidentitysigner + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "errors" + "slices" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/internal/substratex509" + certsv1beta1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/utils/ptr" +) + +// fixedClock is a PassiveClock frozen at a fixed instant. +type fixedClock struct { + now time.Time +} + +func (c fixedClock) Now() time.Time { return c.now } +func (c fixedClock) Since(t time.Time) time.Duration { return c.now.Sub(t) } + +// testNow is a whole-second instant so times survive the x509 encoding +// round-trip (certificates carry 1s precision) and compare exactly. It must +// stay near wall-clock time because GenerateED25519CA stamps CA validity +// from time.Now(). +var testNow = time.Now().UTC().Truncate(time.Second) + +// makePodAndPCR returns a pod and a matching PodCertificateRequest with no +// key material set; callers fill in StubPKCS10Request. +func makePodAndPCR(namespace, podName, serviceAccount string, maxExpirationSeconds int32) (*corev1.Pod, *certsv1beta1.PodCertificateRequest) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: podName, + UID: types.UID("pod-uid-1"), + }, + } + pcr := &certsv1beta1.PodCertificateRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "req-1", + }, + Spec: certsv1beta1.PodCertificateRequestSpec{ + SignerName: Name, + PodName: pod.ObjectMeta.Name, + PodUID: pod.ObjectMeta.UID, + ServiceAccountName: serviceAccount, + ServiceAccountUID: types.UID("sa-uid-1"), + NodeName: types.NodeName("node-1"), + NodeUID: types.UID("node-uid-1"), + MaxExpirationSeconds: ptr.To(maxExpirationSeconds), + }, + } + return pod, pcr +} + +// stubCSR returns a stub PKCS#10 request carrying priv's public key. +func stubCSR(t *testing.T, priv ed25519.PrivateKey) []byte { + t.Helper() + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{}, priv) + if err != nil { + t.Fatalf("while creating stub CSR: %v", err) + } + return csr +} + +func TestMakeCert(t *testing.T) { + testCases := []struct { + name string + namespace string + podName string + serviceAccount string + maxExpirationSeconds int32 + wantLifetime time.Duration + wantURI string + wantEKUs []x509.ExtKeyUsage + wantIdentity *substratex509.PodIdentity + }{ + { + name: "atelet in ate-system", + namespace: "ate-system", + podName: "atelet-abcde", + serviceAccount: "atelet", + maxExpirationSeconds: 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/ate-system/sa/atelet", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid-1", + PodName: "atelet-abcde", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "ordinary workload is client-only", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "requested lifetime capped at 24h", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 7 * 86400, + wantLifetime: 24 * time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + { + name: "shorter requested lifetime honored", + namespace: "default", + podName: "myapp-0", + serviceAccount: "default", + maxExpirationSeconds: 3600, + wantLifetime: time.Hour, + wantURI: "spiffe://cluster.local/ns/default/sa/default", + wantEKUs: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + wantIdentity: &substratex509.PodIdentity{ + Namespace: "default", + ServiceAccountName: "default", + ServiceAccountUID: "sa-uid-1", + PodName: "myapp-0", + PodUID: "pod-uid-1", + NodeName: "node-1", + NodeUID: "node-uid-1", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + subjectPub, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + + pod, pcr := makePodAndPCR(tc.namespace, tc.podName, tc.serviceAccount, tc.maxExpirationSeconds) + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + + kc := fake.NewSimpleClientset(pod, pcr) + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err != nil { + t.Fatalf("MakeCert: %v", err) + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests(tc.namespace).Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching updated PCR: %v", err) + } + if len(gotPCR.Status.Conditions) != 1 || gotPCR.Status.Conditions[0].Type != certsv1beta1.PodCertificateRequestConditionTypeIssued { + t.Fatalf("PCR status not marked Issued: %+v", gotPCR.Status.Conditions) + } + + block, rest := pem.Decode([]byte(gotPCR.Status.CertificateChain)) + if block == nil { + t.Fatalf("certificate chain contains no PEM block") + } + if len(rest) != 0 { + t.Errorf("expected exactly one certificate in chain (no intermediates), got trailing data") + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("while parsing leaf certificate: %v", err) + } + + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: roots, + CurrentTime: testNow, + KeyUsages: tc.wantEKUs, + }); err != nil { + t.Errorf("leaf does not verify against CA root: %v", err) + } + + leafPub, ok := leaf.PublicKey.(ed25519.PublicKey) + if !ok || !leafPub.Equal(subjectPub) { + t.Errorf("leaf public key %v is not the subject key %v", leaf.PublicKey, subjectPub) + } + + wantNotBefore := testNow.Add(-2 * time.Minute) + wantNotAfter := wantNotBefore.Add(tc.wantLifetime) + wantBeginRefreshAt := wantNotAfter.Add(-30 * time.Minute) + if !leaf.NotBefore.Equal(wantNotBefore) { + t.Errorf("got NotBefore %v, want %v", leaf.NotBefore, wantNotBefore) + } + if !leaf.NotAfter.Equal(wantNotAfter) { + t.Errorf("got NotAfter %v, want %v", leaf.NotAfter, wantNotAfter) + } + if gotPCR.Status.NotBefore == nil || !gotPCR.Status.NotBefore.Time.Equal(wantNotBefore) { + t.Errorf("got status NotBefore %v, want %v", gotPCR.Status.NotBefore, wantNotBefore) + } + if gotPCR.Status.NotAfter == nil || !gotPCR.Status.NotAfter.Time.Equal(wantNotAfter) { + t.Errorf("got status NotAfter %v, want %v", gotPCR.Status.NotAfter, wantNotAfter) + } + if gotPCR.Status.BeginRefreshAt == nil || !gotPCR.Status.BeginRefreshAt.Time.Equal(wantBeginRefreshAt) { + t.Errorf("got status BeginRefreshAt %v, want %v", gotPCR.Status.BeginRefreshAt, wantBeginRefreshAt) + } + + if len(leaf.URIs) != 1 || leaf.URIs[0].String() != tc.wantURI { + t.Errorf("got URIs %v, want [%s]", leaf.URIs, tc.wantURI) + } + if !slices.Equal(leaf.ExtKeyUsage, tc.wantEKUs) { + t.Errorf("got EKUs %v, want %v", leaf.ExtKeyUsage, tc.wantEKUs) + } + if !bytes.Equal(leaf.AuthorityKeyId, ca.RootCertificate.SubjectKeyId) { + t.Errorf("got AuthorityKeyId %x, want CA SubjectKeyId %x", leaf.AuthorityKeyId, ca.RootCertificate.SubjectKeyId) + } + + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil { + t.Fatalf("while extracting PodIdentity: %v", err) + } + if *identity != *tc.wantIdentity { + t.Errorf("got PodIdentity %+v, want %+v", identity, tc.wantIdentity) + } + }) + } +} + +func TestMakeCertErrors(t *testing.T) { + testCases := []struct { + name string + omitPod bool + podUID types.UID + omitKey bool + failUpdate bool + }{ + { + name: "pod not found", + omitPod: true, + podUID: "pod-uid-1", + }, + { + name: "pod UID mismatch", + podUID: "other-uid", + }, + { + name: "no key material in PCR", + podUID: "pod-uid-1", + omitKey: true, + }, + { + name: "status update fails", + podUID: "pod-uid-1", + failUpdate: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400) + pod.ObjectMeta.UID = tc.podUID + if !tc.omitKey { + _, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + } + + objects := []runtime.Object{pcr} + if !tc.omitPod { + objects = append(objects, pod) + } + kc := fake.NewSimpleClientset(objects...) + if tc.failUpdate { + kc.PrependReactor("update", "podcertificaterequests", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("injected update failure") + }) + } + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err == nil { + t.Fatalf("MakeCert: got nil error, want error") + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests("ate-system").Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching PCR: %v", err) + } + if len(gotPCR.Status.Conditions) != 0 || gotPCR.Status.CertificateChain != "" { + t.Errorf("PCR status updated despite error: %+v", gotPCR.Status) + } + }) + } +} + +func TestMakeCertChainIncludesIntermediates(t *testing.T) { + ca, err := localca.GenerateED25519CA("test-ca") + if err != nil { + t.Fatalf("while generating CA: %v", err) + } + intermediateCA, err := localca.GenerateED25519CA("test-intermediate") + if err != nil { + t.Fatalf("while generating intermediate CA: %v", err) + } + ca.IntermediateCertificates = []*x509.Certificate{intermediateCA.RootCertificate} + caPool := &localca.Pool{CAs: []*localca.CA{ca}} + + _, subjectPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("while generating subject key: %v", err) + } + + pod, pcr := makePodAndPCR("ate-system", "atelet-abcde", "atelet", 86400) + pcr.Spec.StubPKCS10Request = stubCSR(t, subjectPriv) + + kc := fake.NewSimpleClientset(pod, pcr) + impl := NewImpl(kc, caPool, fixedClock{now: testNow}) + + if err := impl.MakeCert(context.Background(), pcr); err != nil { + t.Fatalf("MakeCert: %v", err) + } + + gotPCR, err := kc.CertificatesV1beta1().PodCertificateRequests("ate-system").Get(context.Background(), "req-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("while fetching updated PCR: %v", err) + } + + var chainDER [][]byte + rest := []byte(gotPCR.Status.CertificateChain) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + chainDER = append(chainDER, block.Bytes) + } + if len(rest) != 0 { + t.Errorf("certificate chain has trailing non-PEM data") + } + if len(chainDER) != 2 { + t.Fatalf("got %d certificates in chain, want 2 (leaf + intermediate)", len(chainDER)) + } + if _, err := x509.ParseCertificate(chainDER[0]); err != nil { + t.Errorf("while parsing leaf certificate: %v", err) + } + if !bytes.Equal(chainDER[1], intermediateCA.RootCertificate.Raw) { + t.Errorf("second chain entry is not the intermediate certificate") + } +} + +func TestDesiredClusterTrustBundles(t *testing.T) { + ca1, err := localca.GenerateED25519CA("test-ca-1") + if err != nil { + t.Fatalf("while generating CA 1: %v", err) + } + ca2, err := localca.GenerateED25519CA("test-ca-2") + if err != nil { + t.Fatalf("while generating CA 2: %v", err) + } + caPool := &localca.Pool{CAs: []*localca.CA{ca1, ca2}} + impl := NewImpl(nil, caPool, fixedClock{now: testNow}) + + ctbs := impl.DesiredClusterTrustBundles() + if len(ctbs) != 1 { + t.Fatalf("got %d ClusterTrustBundles, want 1", len(ctbs)) + } + ctb := ctbs[0] + + if want := CTBPrefix + "primary-bundle"; ctb.ObjectMeta.Name != want { + t.Errorf("got CTB name %q, want %q", ctb.ObjectMeta.Name, want) + } + if ctb.Spec.SignerName != Name { + t.Errorf("got signer name %q, want %q", ctb.Spec.SignerName, Name) + } + if got := ctb.ObjectMeta.Labels["podcert.ate.dev/canarying"]; got != "live" { + t.Errorf("got canarying label %q, want %q", got, "live") + } + + wantBundle := &bytes.Buffer{} + for _, ca := range caPool.CAs { + wantBundle.Write(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: ca.RootCertificate.Raw, + })) + } + if ctb.Spec.TrustBundle != wantBundle.String() { + t.Errorf("got trust bundle:\n%s\nwant:\n%s", ctb.Spec.TrustBundle, wantBundle.String()) + } +} diff --git a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go index 4258f1921..1a995b20d 100644 --- a/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go +++ b/cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go @@ -135,6 +135,13 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq } } + // This is returned as a transient error to allow retries. + // Without this, ate-apiserver can have a servicedns cert without DNS name + // while the covering Service is being created, and cache it for 24 hours. + if len(dnsNames) == 0 { + return fmt.Errorf("pod %s/%s is not (yet) selected by any Service; refusing to issue a serving cert with no DNS SANs", pcr.ObjectMeta.Namespace, pcr.Spec.PodName) + } + // TODO: Encode the OIDC issuer of the cluster into the certificate. subjectPublicKey, err := podcertificate.PublicKey(pcr) @@ -156,6 +163,7 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq notAfter := notBefore.Add(lifetime) beginRefreshAt := notAfter.Add(-30 * time.Minute) + parent := h.caPool.CAs[0].RootCertificate template := &x509.Certificate{ BasicConstraintsValid: true, NotBefore: notBefore, @@ -163,9 +171,12 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq DNSNames: dnsNames, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + // Link the leaf to its issuing CA by key id. Needed this for Valkey + // to understand which CA to use when validating a client cert. + AuthorityKeyId: parent.SubjectKeyId, } - subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, h.caPool.CAs[0].RootCertificate, subjectPublicKey, h.caPool.CAs[0].SigningKey) + subjectCertDER, err := x509.CreateCertificate(rand.Reader, template, parent, subjectPublicKey, h.caPool.CAs[0].SigningKey) if err != nil { return fmt.Errorf("while signing subject cert: %w", err) } diff --git a/hack/install-ate.sh b/hack/install-ate.sh index c8f568975..78429e3eb 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -63,7 +63,7 @@ function usage() { echo " --deploy-ate-system Deploy core system (CRDs, atelet, apiserver)" echo " --delete-ate-system Delete core system" echo " --delete-all Delete core system and all registered demos" - echo " --auth-mode=mtls|jwt Select ateapi auth mode for --deploy-ate-system (default: mtls)" + echo " --ateapi-client-auth=cert|token Select how in-cluster clients authenticate to ateapi for --deploy-ate-system (default: cert; the server always accepts both)" echo "" echo "Infrastructure components:" echo "" @@ -133,26 +133,26 @@ run_ko() { esac } -ate_auth_mode() { - case "${ATE_API_AUTH_MODE:-mtls}" in - mtls|jwt) - echo "${ATE_API_AUTH_MODE:-mtls}" +ateapi_client_auth() { + case "${ATE_ATEAPI_CLIENT_AUTH:-cert}" in + cert|token) + echo "${ATE_ATEAPI_CLIENT_AUTH:-cert}" ;; *) - echo "Error: ATE_API_AUTH_MODE must be mtls or jwt, got '${ATE_API_AUTH_MODE}'" >&2 + echo "Error: ATE_ATEAPI_CLIENT_AUTH must be cert or token, got '${ATE_ATEAPI_CLIENT_AUTH}'" >&2 exit 1 ;; esac } render_ate_system_manifests() { - local auth_mode="" - auth_mode="$(ate_auth_mode)" + local client_auth="" + client_auth="$(ateapi_client_auth)" - if [[ "${auth_mode}" == "jwt" ]]; then - local overlay="manifests/ate-install/jwt" + if [[ "${client_auth}" == "token" ]]; then + local overlay="manifests/ate-install/token-client" if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then - overlay="manifests/ate-install/kind-jwt" + overlay="manifests/ate-install/kind-token-client" fi kubectl kustomize "${overlay}" --load-restrictor LoadRestrictionsNone | run_ko resolve -f - return @@ -167,17 +167,36 @@ render_ate_system_manifests() { fi } -create_valkey_ca_certs_secret() { - log_step "create_valkey_ca_certs_secret" - local ca_certs="" - # Extract from in-cluster service-dns-ca-pool secret (base64 json) +# Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate. +ca_pool_root_pem() { + local secret="$1" local pool_json="" - pool_json=$(run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool -o jsonpath='{.data.pool}' | base64 --decode) - # Extract RootCertificateDER base64 string + pool_json=$(run_kubectl get secret -n podcertificate-controller-system "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) local der_base64="" der_base64=$(echo "${pool_json}" | grep -o '"RootCertificateDER":"[^"]*' | sed 's/"RootCertificateDER":"//') - # Convert DER to PEM certificate - ca_certs=$(echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem) + echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem +} + +create_valkey_ca_certs_secret() { + log_step "create_valkey_ca_certs_secret" + # valkey requires a single tls-ca-cert-file to verify client and server certs it sees, + # so it needs both CAs: + # - servicedns CA: verifies valkey peers' server certs. + # - podidentity CA: verifies the client certs that connect to valkey + # (apiserver, the init job, and peers acting as clients). + # Extract each root into its own variable: errexit cannot see a substitution + # failing inside printf's argument list, which would silently produce a CA + # file with a missing root. + local servicedns_root="" + servicedns_root=$(ca_pool_root_pem service-dns-ca-pool) + local podidentity_root="" + podidentity_root=$(ca_pool_root_pem pod-identity-ca-pool) + if [[ -z "${servicedns_root}" || -z "${podidentity_root}" ]]; then + echo "error: failed to extract a CA root for valkey-ca-certs" >&2 + return 1 + fi + local ca_certs="" + ca_certs=$(printf '%s\n%s\n' "${servicedns_root}" "${podidentity_root}") run_kubectl create secret generic valkey-ca-certs \ --from-literal=ca.crt="${ca_certs}" \ @@ -227,7 +246,9 @@ create_api_server_env_vars() { redis_address="valkey-cluster.ate-system.svc:6379" use_iam_auth="false" tls_server_name="valkey-cluster.ate-system.svc" - client_cert="/run/servicedns.podcert.ate.dev/credential-bundle.pem" + # The apiserver dials valkey as a client, so it presents a podidentity + # (SPIFFE) client cert rather than a servicedns serving cert. + client_cert="/run/podidentity.podcert.ate.dev/credential-bundle.pem" echo "REDIS_ADDRESS: ${redis_address}" @@ -530,13 +551,13 @@ BENCHMARK_WORKER_COUNT=1 prescan_args=("$@") for ((i = 0; i < ${#prescan_args[@]}; i++)); do case "${prescan_args[i]}" in - --auth-mode=*) ATE_API_AUTH_MODE="${prescan_args[i]#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${prescan_args[i]#*=}" ;; + --ateapi-client-auth) if (( i + 1 >= ${#prescan_args[@]} )); then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi - ATE_API_AUTH_MODE="${prescan_args[$((i + 1))]}" + ATE_ATEAPI_CLIENT_AUTH="${prescan_args[$((i + 1))]}" ;; --benchmark-worker-count) BENCHMARK_WORKER_COUNT="${prescan_args[i+1]:-1}" @@ -561,14 +582,14 @@ while [[ "$#" -gt 0 ]]; do done case $1 in - --auth-mode=*) ATE_API_AUTH_MODE="${1#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${1#*=}" ;; + --ateapi-client-auth) shift if [[ "$#" -eq 0 ]]; then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi - ATE_API_AUTH_MODE="$1" + ATE_ATEAPI_CLIENT_AUTH="$1" ;; --deploy-ate-system) deploy_ate_system ;; diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 2e29f6ba6..1f92da24a 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -48,18 +48,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -ATE_API_AUTH_MODE="${ATE_API_AUTH_MODE:-mtls}" +ATE_ATEAPI_CLIENT_AUTH="${ATE_ATEAPI_CLIENT_AUTH:-cert}" while [[ $# -gt 0 ]]; do case "$1" in - --auth-mode=*) ATE_API_AUTH_MODE="${1#*=}" ;; - --auth-mode) + --ateapi-client-auth=*) ATE_ATEAPI_CLIENT_AUTH="${1#*=}" ;; + --ateapi-client-auth) if [[ $# -lt 2 ]]; then - echo "Error: --auth-mode requires mtls or jwt" >&2 + echo "Error: --ateapi-client-auth requires cert or token" >&2 exit 1 fi shift - ATE_API_AUTH_MODE="$1" + ATE_ATEAPI_CLIENT_AUTH="$1" ;; *) echo "Error: unknown argument $1" >&2 @@ -69,10 +69,10 @@ while [[ $# -gt 0 ]]; do shift done -case "${ATE_API_AUTH_MODE}" in - mtls|jwt) ;; +case "${ATE_ATEAPI_CLIENT_AUTH}" in + cert|token) ;; *) - echo "Error: --auth-mode must be mtls or jwt, got '${ATE_API_AUTH_MODE}'" >&2 + echo "Error: --ateapi-client-auth must be cert or token, got '${ATE_ATEAPI_CLIENT_AUTH}'" >&2 exit 1 ;; esac @@ -131,10 +131,10 @@ fi log "Deploying the ate control plane (--deploy-ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --ateapi-client-auth="${ATE_ATEAPI_CLIENT_AUTH}" fi # --- 4. apply the demo ------------------------------------------------------ diff --git a/internal/ateapiauth/client.go b/internal/ateapiauth/client.go index 8ae0807fe..aed3b5d71 100644 --- a/internal/ateapiauth/client.go +++ b/internal/ateapiauth/client.go @@ -22,6 +22,7 @@ import ( "os" "strings" + "github.com/agent-substrate/substrate/internal/credbundle" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) @@ -31,27 +32,33 @@ const ( DefaultServiceAccountTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" ) -// ClientConfig configures how to dial the ateapi gRPC server. -// -// - Mode=ModeMTLS: insecure TLS dial (InsecureSkipVerify=true). Client -// identity is expected to come from mTLS credentials projected into -// the pod (servicedns.podcert.ate.dev). No app-level credentials. -// - Mode=ModeJWT: validates the server cert against CAFile, sends a Bearer -// token from TokenFile as per-RPC credentials. +// ClientConfig configures how to dial the ateapi gRPC server. The server +// cert is always validated against CAFile. UseTokenAuth selects the client +// credential: a client certificate from ClientCredBundle (mutual TLS, +// re-read on every handshake so in-place pod-certificate rotations are +// picked up) by default, or a Bearer token from TokenFile sent as per-RPC +// credentials. The path not selected is ignored. type ClientConfig struct { - Mode Mode + // UseTokenAuth authenticates with the Bearer token from TokenFile instead + // of the client certificate from ClientCredBundle. + UseTokenAuth bool // CAFile is a PEM file containing CA certs that sign the server cert. - // Required in all modes. Ignored for ModeMTLS until mTLS verification is - // fully wired. + // Required. CAFile string // ServerName overrides SNI / hostname verification. Optional. ServerName string // TokenFile is a path to a Kubernetes projected ServiceAccount token used - // as a Bearer credential. Required for ModeJWT. + // as a Bearer credential. Required when UseTokenAuth is set, ignored + // otherwise. TokenFile string + + // ClientCredBundle is a PEM file containing the client certificate chain + // and PKCS8 private key presented to the server. Required unless + // UseTokenAuth is set, ignored otherwise. + ClientCredBundle string } // DialOptions returns the grpc.DialOption set described by cfg, suitable to @@ -60,38 +67,44 @@ func DialOptions(cfg ClientConfig) ([]grpc.DialOption, error) { if cfg.CAFile == "" { return nil, fmt.Errorf("ateapiauth: CAFile is required") } - switch cfg.Mode { - case "", ModeMTLS: - tlsCfg := &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit opt-in - return []grpc.DialOption{ - grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), - }, nil - - case ModeJWT: + if cfg.UseTokenAuth { if cfg.TokenFile == "" { - return nil, fmt.Errorf("ateapiauth: jwt mode requires TokenFile") - } - caPEM, err := os.ReadFile(cfg.CAFile) - if err != nil { - return nil, fmt.Errorf("ateapiauth: reading CA file: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - return nil, fmt.Errorf("ateapiauth: no certificates found in CA file %q", cfg.CAFile) - } - tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: pool, - ServerName: cfg.ServerName, + return nil, fmt.Errorf("ateapiauth: token auth requires a token file") } + } else if cfg.ClientCredBundle == "" { + return nil, fmt.Errorf("ateapiauth: a client credential bundle (mTLS) is required unless token auth is enabled") + } + pool, err := loadCAPool(cfg.CAFile) + if err != nil { + return nil, err + } + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + RootCAs: pool, + ServerName: cfg.ServerName, + } + if cfg.UseTokenAuth { return []grpc.DialOption{ grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), grpc.WithPerRPCCredentials(&fileTokenCreds{path: cfg.TokenFile}), }, nil + } + tlsCfg.GetClientCertificate = credbundle.ClientLoader(cfg.ClientCredBundle) + return []grpc.DialOption{ + grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), + }, nil +} - default: - return nil, fmt.Errorf("ateapiauth: unknown client mode %q", cfg.Mode) +func loadCAPool(caFile string) (*x509.CertPool, error) { + caPEM, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("ateapiauth: reading CA file: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("ateapiauth: no certificates found in CA file %q", caFile) } + return pool, nil } // fileTokenCreds reads a Kubernetes projected SA token from disk for every diff --git a/internal/ateapiauth/client_test.go b/internal/ateapiauth/client_test.go index d8f271ee3..9109699d2 100644 --- a/internal/ateapiauth/client_test.go +++ b/internal/ateapiauth/client_test.go @@ -14,18 +14,279 @@ package ateapiauth -import "testing" +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) func TestDialOptionsRequiresCAFile(t *testing.T) { - for _, mode := range []Mode{ModeMTLS, ModeJWT} { - t.Run(string(mode), func(t *testing.T) { - _, err := DialOptions(ClientConfig{ - Mode: mode, - TokenFile: "token", - }) + for name, cfg := range map[string]ClientConfig{ + "token": {UseTokenAuth: true, TokenFile: "token"}, + "cert": {ClientCredBundle: "bundle.pem"}, + } { + t.Run(name, func(t *testing.T) { + _, err := DialOptions(cfg) if err == nil { t.Fatalf("DialOptions() error = nil, want error") } }) } } + +func TestDialOptionsRequiresModeCredential(t *testing.T) { + for name, cfg := range map[string]ClientConfig{ + "cert mode without bundle": {CAFile: "ca.pem"}, + "token mode without token": {CAFile: "ca.pem", UseTokenAuth: true}, + "cert path does not satisfy token mode": {CAFile: "ca.pem", UseTokenAuth: true, ClientCredBundle: "bundle.pem"}, + } { + t.Run(name, func(t *testing.T) { + _, err := DialOptions(cfg) + if err == nil { + t.Fatalf("DialOptions() error = nil, want error") + } + }) + } +} + +// TestDialOptionsMTLSHandshake dials a server that requires and verifies +// client certificates — the configuration ateapi will move to — and checks +// that DialOptions with a client credential bundle completes the handshake, +// while a certificate-less client is rejected. +func TestDialOptionsMTLSHandshake(t *testing.T) { + ca := newTestCA(t) + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.pem") + writeFile(t, caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.certDER})) + + clientBundle := filepath.Join(dir, "client-bundle.pem") + writeFile(t, clientBundle, ca.issueClientBundle(t, "spiffe://cluster.local/ns/ate-system/sa/ate-controller")) + + serverCert := ca.issueServerCert(t) + caPool := x509.NewCertPool() + caPool.AddCert(ca.cert) + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: caPool, + MinVersion: tls.VersionTLS13, + }))) + healthpb.RegisterHealthServer(srv, health.NewServer()) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + go srv.Serve(lis) + defer srv.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + t.Run("with client cert", func(t *testing.T) { + // TokenFile is ignored in cert mode (the default). + opts, err := DialOptions(ClientConfig{ + CAFile: caFile, + ClientCredBundle: clientBundle, + TokenFile: filepath.Join(dir, "does-not-exist-token"), + }) + if err != nil { + t.Fatalf("DialOptions() error = %v", err) + } + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code != codes.OK { + t.Fatalf("health check code = %v, want %v", code, codes.OK) + } + }) + + t.Run("without client cert is rejected", func(t *testing.T) { + opts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + RootCAs: caPool, + MinVersion: tls.VersionTLS13, + }))} + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code == codes.OK { + t.Fatalf("health check code = %v, want handshake failure", code) + } + }) +} + +// TestDialOptionsTokenSendsBearer dials a server that accepts certless +// clients — ateapi's current configuration — and checks that DialOptions +// with a token file attaches the token as an `authorization: Bearer` header +// on every RPC. +func TestDialOptionsTokenSendsBearer(t *testing.T) { + ca := newTestCA(t) + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.pem") + writeFile(t, caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.certDER})) + tokenFile := filepath.Join(dir, "token") + writeFile(t, tokenFile, []byte("test-token\n")) + + gotAuth := make(chan string, 1) + srv := grpc.NewServer( + grpc.Creds(credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{ca.issueServerCert(t)}, + ClientAuth: tls.VerifyClientCertIfGiven, + MinVersion: tls.VersionTLS13, + })), + grpc.UnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, _ := metadata.FromIncomingContext(ctx) + gotAuth <- strings.Join(md.Get("authorization"), ",") + return handler(ctx, req) + }), + ) + healthpb.RegisterHealthServer(srv, health.NewServer()) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + go srv.Serve(lis) + defer srv.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // ClientCredBundle stays set (as the base manifests leave it) but is + // ignored in token mode — it doesn't even need to exist on disk. + opts, err := DialOptions(ClientConfig{ + CAFile: caFile, + UseTokenAuth: true, + TokenFile: tokenFile, + ClientCredBundle: filepath.Join(dir, "does-not-exist.pem"), + }) + if err != nil { + t.Fatalf("DialOptions() error = %v", err) + } + if code := healthCheckCode(ctx, t, lis.Addr().String(), opts); code != codes.OK { + t.Fatalf("health check code = %v, want %v", code, codes.OK) + } + if got, want := <-gotAuth, "Bearer test-token"; got != want { + t.Errorf("authorization header = %q, want %q", got, want) + } +} + +func healthCheckCode(ctx context.Context, t *testing.T, target string, opts []grpc.DialOption) codes.Code { + t.Helper() + conn, err := grpc.NewClient(target, opts...) + if err != nil { + t.Fatalf("grpc.NewClient() error = %v", err) + } + defer conn.Close() + _, err = healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}) + return status.Code(err) +} + +type testCA struct { + cert *x509.Certificate + certDER []byte + key *ecdsa.PrivateKey +} + +func newTestCA(t *testing.T) *testCA { + t.Helper() + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("create CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse CA certificate: %v", err) + } + return &testCA{cert: cert, certDER: der, key: key} +} + +// issueClientBundle returns a PEM credential bundle (leaf certificate + PKCS8 +// private key) for a client certificate carrying the given SPIFFE URI SAN. +func (ca *testCA) issueClientBundle(t *testing.T, spiffeID string) []byte { + t.Helper() + uri, err := url.Parse(spiffeID) + if err != nil { + t.Fatalf("parse SPIFFE ID: %v", err) + } + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "test-client"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + URIs: []*url.URL{uri}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("create client certificate: %v", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal PKCS8 key: %v", err) + } + return append( + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})..., + ) +} + +func (ca *testCA) issueServerCert(t *testing.T) tls.Certificate { + t.Helper() + key := generateKey(t) + template := &x509.Certificate{ + SerialNumber: big.NewInt(3), + Subject: pkix.Name{CommonName: "test-server"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("create server certificate: %v", err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +func generateKey(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + return key +} + +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/internal/ateapiauth/server.go b/internal/ateapiauth/server.go index b8e5167bf..b2891c9da 100644 --- a/internal/ateapiauth/server.go +++ b/internal/ateapiauth/server.go @@ -12,75 +12,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ateapiauth adds optional Kubernetes ServiceAccount JWT -// authentication on top of the ateapi gRPC server, and a matching client -// dial helper. It does not replace the existing TLS / mTLS path — the -// server's transport credentials still apply unchanged. Set Mode=ModeJWT -// on the server to require an `authorization: Bearer ` header -// on every RPC; Mode=ModeMTLS (the default) leaves identity to the -// transport-layer mTLS credentials. +// Package ateapiauth authenticates clients of the ateapi gRPC server, and +// provides a matching client dial helper. The server interceptor takes +// identity from the transport-layer mTLS credentials when the client +// presented a certificate, and otherwise requires an authorization +// header `Bearer `. Requests with no credentials are rejected. package ateapiauth import ( "context" "fmt" + "log/slog" "strings" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" ) -// Mode selects whether the JWT interceptor enforces a Bearer token. -type Mode string - -const ( - ModeMTLS Mode = "mtls" - ModeJWT Mode = "jwt" -) - -// ParseMode parses a flag value into a Mode, defaulting to ModeMTLS on empty. -// ModeMTLS means identity is established by the transport-layer mTLS -// credentials; the interceptor performs no app-level checks. ModeJWT -// additionally requires a Kubernetes SA Bearer token on every RPC. -func ParseMode(s string) (Mode, error) { - switch Mode(s) { - case "", ModeMTLS: - return ModeMTLS, nil - case ModeJWT: - return ModeJWT, nil - default: - return "", fmt.Errorf("unknown auth mode %q (want mtls|jwt)", s) - } -} - func ValidateServerConfig(cfg ServerConfig) error { - switch cfg.Mode { - case "", ModeMTLS: - return nil - case ModeJWT: - if cfg.VerifyBearerToken == nil { - return fmt.Errorf("jwt mode requires bearer token verifier") - } - return nil - default: - return fmt.Errorf("unknown auth mode %q", cfg.Mode) + if cfg.VerifyBearerToken == nil { + return fmt.Errorf("a bearer token verifier is required") } + return nil } // ServerConfig configures the server-side auth interceptor. type ServerConfig struct { - Mode Mode - - // VerifyBearerToken verifies a Bearer token presented by a client. Required - // for ModeJWT and ignored for ModeMTLS. - VerifyBearerToken func(context.Context, string) error + // VerifyBearerToken verifies a Bearer token presented by a client and + // returns the authenticated principal's ID (e.g. the JWT subject). It + // authenticates clients that did not present a certificate identity + // (e.g. kubectl-ate, which dials without a client certificate). + VerifyBearerToken func(context.Context, string) (string, error) } // UnaryServerInterceptor returns a gRPC unary interceptor enforcing cfg. func UnaryServerInterceptor(cfg ServerConfig) grpc.UnaryServerInterceptor { - auth := serverAuthenticatorFor(cfg) + auth := newChainedAuthenticator(cfg) return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { newCtx, err := auth.authenticate(ctx) if err != nil { @@ -92,7 +64,7 @@ func UnaryServerInterceptor(cfg ServerConfig) grpc.UnaryServerInterceptor { // StreamServerInterceptor returns a gRPC stream interceptor enforcing cfg. func StreamServerInterceptor(cfg ServerConfig) grpc.StreamServerInterceptor { - auth := serverAuthenticatorFor(cfg) + auth := newChainedAuthenticator(cfg) return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { newCtx, err := auth.authenticate(ss.Context()) if err != nil { @@ -109,33 +81,69 @@ type wrappedStream struct { func (w *wrappedStream) Context() context.Context { return w.ctx } -type serverAuthenticator interface { - authenticate(context.Context) (context.Context, error) -} - -func serverAuthenticatorFor(cfg ServerConfig) serverAuthenticator { - switch cfg.Mode { - case "", ModeMTLS: - return mtlsServerAuthenticator{} - case ModeJWT: - return jwtServerAuthenticator{ +func newChainedAuthenticator(cfg ServerConfig) chainedServerAuthenticator { + return chainedServerAuthenticator{ + jwt: jwtServerAuthenticator{ verifyBearerToken: cfg.VerifyBearerToken, - } + }, } +} + +// chainedServerAuthenticator first checks mTLS peer, then checks +// a bearer token in the header. +type chainedServerAuthenticator struct { + // jwt authenticates clients that did not present a certificate identity + // (e.g. kubectl-ate, which dials without a client certificate). + jwt jwtServerAuthenticator +} - return invalidServerAuthenticator{mode: cfg.Mode} +func (a chainedServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { + if id, ok := mtlsPeerIdentity(ctx); ok { + return principal.InjectContext(ctx, principal.PrincipalInfo{ + ID: id, + Kind: principal.KindMTLS, + }), nil + } + return a.jwt.authenticate(ctx) } -type mtlsServerAuthenticator struct{} +// mtlsPeerIdentity extracts the client identity (the first URI SAN, a SPIFFE +// ID) from the transport-authenticated peer certificate. +func mtlsPeerIdentity(ctx context.Context) (string, bool) { + p, ok := peer.FromContext(ctx) + if !ok || p.AuthInfo == nil { + slog.DebugContext(ctx, "No mTLS peer identity: no peer or auth info in context.") + return "", false + } -func (mtlsServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { - // TODO: Extract the transport-authenticated client identity and attach it - // to ctx once ateapi has an authorization layer. - return ctx, nil + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok { + slog.DebugContext(ctx, "No mTLS peer identity: no TLS info in context.") + return "", false + } + + if len(tlsInfo.State.PeerCertificates) == 0 { + slog.DebugContext(ctx, "No mTLS peer identity: no peer certificates in TLS info.") + return "", false + } + + clientCert := tlsInfo.State.PeerCertificates[0] + if len(clientCert.URIs) == 0 { + slog.DebugContext(ctx, "No mTLS peer identity: no URIs in peer certificate.") + return "", false + } + + id := clientCert.URIs[0].String() + if id == "" { + slog.DebugContext(ctx, "No mTLS peer identity: client cert URI is empty string") + return "", false + } + slog.InfoContext(ctx, "Authentication successful", slog.String("id", id)) + return id, true } type jwtServerAuthenticator struct { - verifyBearerToken func(context.Context, string) error + verifyBearerToken func(context.Context, string) (string, error) } func (a jwtServerAuthenticator) authenticate(ctx context.Context) (context.Context, error) { @@ -143,20 +151,14 @@ func (a jwtServerAuthenticator) authenticate(ctx context.Context) (context.Conte if !ok { return nil, status.Error(codes.Unauthenticated, "missing bearer token") } - if err := a.verifyBearerToken(ctx, bearer); err != nil { + id, err := a.verifyBearerToken(ctx, bearer) + if err != nil { return nil, status.Errorf(codes.Unauthenticated, "invalid bearer token: %v", err) } - // TODO: Attach the verified JWT identity to ctx once ateapi has an - // authorization layer that consumes it. - return ctx, nil -} - -type invalidServerAuthenticator struct { - mode Mode -} - -func (a invalidServerAuthenticator) authenticate(context.Context) (context.Context, error) { - return nil, status.Errorf(codes.Internal, "invalid auth mode %q", a.mode) + return principal.InjectContext(ctx, principal.PrincipalInfo{ + ID: id, + Kind: principal.KindJWT, + }), nil } func bearerToken(ctx context.Context) (string, bool) { diff --git a/internal/ateapiauth/server_test.go b/internal/ateapiauth/server_test.go index 5a0c0631e..e9538b5d8 100644 --- a/internal/ateapiauth/server_test.go +++ b/internal/ateapiauth/server_test.go @@ -16,49 +16,29 @@ package ateapiauth import ( "context" + "crypto/tls" + "crypto/x509" "fmt" + "net/url" "testing" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" ) -func TestParseMode(t *testing.T) { - cases := []struct { - in string - want Mode - wantErr bool - }{ - {"", ModeMTLS, false}, - {"mtls", ModeMTLS, false}, - {"jwt", ModeJWT, false}, - {"none", "", true}, - {"bogus", "", true}, - } - for _, tc := range cases { - got, err := ParseMode(tc.in) - if (err != nil) != tc.wantErr { - t.Errorf("ParseMode(%q) err=%v wantErr=%v", tc.in, err, tc.wantErr) - } - if !tc.wantErr && got != tc.want { - t.Errorf("ParseMode(%q)=%v want %v", tc.in, got, tc.want) - } - } -} - func TestValidateServerConfig(t *testing.T) { tests := []struct { name string cfg ServerConfig wantErr bool }{ - {name: "mtls zero config", cfg: ServerConfig{Mode: ModeMTLS}}, - {name: "empty mode zero config", cfg: ServerConfig{}}, - {name: "jwt valid", cfg: ServerConfig{Mode: ModeJWT, VerifyBearerToken: func(context.Context, string) error { return nil }}}, - {name: "jwt missing verifier", cfg: ServerConfig{Mode: ModeJWT}, wantErr: true}, - {name: "unknown mode", cfg: ServerConfig{Mode: Mode("bogus")}, wantErr: true}, + {name: "valid", cfg: ServerConfig{VerifyBearerToken: func(context.Context, string) (string, error) { return "", nil }}}, + {name: "missing verifier", cfg: ServerConfig{}, wantErr: true}, } for _, tt := range tests { @@ -71,17 +51,117 @@ func TestValidateServerConfig(t *testing.T) { } } -func TestMTLSServerAuthenticatorAllowsAnonymous(t *testing.T) { - _, err := (mtlsServerAuthenticator{}).authenticate(context.Background()) - if err != nil { - t.Fatalf("ModeMTLS should not error: %v", err) +func TestChainedServerAuthenticatorPrincipal(t *testing.T) { + const subject = "system:serviceaccount:ate-system:ate-client" + spiffeID := &url.URL{Scheme: "spiffe", Host: "ate.dev", Path: "/ns/default/sa/router"} + spiffePeer := func(ctx context.Context) context.Context { + return peer.NewContext(ctx, &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{spiffeID}}}, + }}, + }) + } + withBearer := func(ctx context.Context, token string) context.Context { + return metadata.NewIncomingContext(ctx, metadata.Pairs("authorization", "Bearer "+token)) + } + verifyGoodToken := func(_ context.Context, bearer string) (string, error) { + if bearer != "good-token" { + return "", fmt.Errorf("bad token") + } + return subject, nil + } + + tests := []struct { + name string + ctx context.Context + // verify is the bearer token verifier; nil means the test fails if + // it is called (the certificate identity must take precedence). + verify func(context.Context, string) (string, error) + want principal.PrincipalInfo + wantCode codes.Code + }{ + { + name: "no peer and no token", + ctx: context.Background(), + wantCode: codes.Unauthenticated, + }, + { + name: "peer without certificates and no token", + ctx: peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{}, + }), + wantCode: codes.Unauthenticated, + }, + { + name: "no peer with valid bearer", + ctx: withBearer(context.Background(), "good-token"), + verify: verifyGoodToken, + want: principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT}, + wantCode: codes.OK, + }, + { + name: "no peer with invalid bearer", + ctx: withBearer(context.Background(), "bad-token"), + verify: verifyGoodToken, + wantCode: codes.Unauthenticated, + }, + { + name: "certificate without URI SAN with valid bearer", + ctx: withBearer(peer.NewContext(context.Background(), &peer.Peer{ + AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{{}}, + }}, + }), "good-token"), + verify: verifyGoodToken, + want: principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT}, + wantCode: codes.OK, + }, + { + name: "certificate with SPIFFE URI SAN", + ctx: spiffePeer(context.Background()), + want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + wantCode: codes.OK, + }, + { + name: "certificate takes precedence over bearer", + ctx: withBearer(spiffePeer(context.Background()), "good-token"), + want: principal.PrincipalInfo{ID: spiffeID.String(), Kind: principal.KindMTLS}, + wantCode: codes.OK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + verify := tt.verify + if verify == nil { + verify = func(context.Context, string) (string, error) { + t.Fatal("bearer token verifier called; certificate identity must take precedence") + return "", nil + } + } + auth := newChainedAuthenticator(ServerConfig{VerifyBearerToken: verify}) + newCtx, err := auth.authenticate(tt.ctx) + if code := status.Code(err); code != tt.wantCode { + t.Fatalf("authenticate: code=%v (err=%v), want %v", code, err, tt.wantCode) + } + if tt.wantCode != codes.OK { + return + } + got, ok := principal.FromContext(newCtx) + if !ok { + t.Fatal("no principal in context") + } + if got != tt.want { + t.Errorf("principal=%+v want %+v", got, tt.want) + } + }) } } func TestJWTServerAuthenticatorRequiresBearer(t *testing.T) { auth := jwtServerAuthenticator{ - verifyBearerToken: func(context.Context, string) error { - return fmt.Errorf("bad token") + verifyBearerToken: func(context.Context, string) (string, error) { + return "", fmt.Errorf("bad token") }, } @@ -99,6 +179,29 @@ func TestJWTServerAuthenticatorRequiresBearer(t *testing.T) { } } +func TestJWTServerAuthenticatorInjectsPrincipal(t *testing.T) { + const subject = "system:serviceaccount:default:router" + auth := jwtServerAuthenticator{ + verifyBearerToken: func(context.Context, string) (string, error) { + return subject, nil + }, + } + + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer good-token")) + newCtx, err := auth.authenticate(ctx) + if err != nil { + t.Fatalf("authenticate: %v", err) + } + got, ok := principal.FromContext(newCtx) + if !ok { + t.Fatal("no principal in context") + } + want := principal.PrincipalInfo{ID: subject, Kind: principal.KindJWT} + if got != want { + t.Errorf("principal=%+v want %+v", got, want) + } +} + func TestBearerToken(t *testing.T) { cases := []struct { name string diff --git a/internal/ateclient/builder.go b/internal/ateclient/builder.go index d8a8fc9bd..1bfe0cf2f 100644 --- a/internal/ateclient/builder.go +++ b/internal/ateclient/builder.go @@ -21,7 +21,6 @@ import ( "io" "net/http" "os" - "strings" "sync" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -77,7 +76,7 @@ func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, var cli *Client if endpoint != "" { - cli, err = dialDirect(kubeconfigPath, k8sContext, endpoint, traceEnabled) + cli, err = dialDirect(ctx, kubeconfigPath, k8sContext, endpoint, traceEnabled) } else { cli, err = dialPortForward(ctx, kubeconfigPath, k8sContext, traceEnabled) } @@ -91,13 +90,23 @@ func NewClient(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, return cli, nil } -func dialDirect(kubeconfigPath, k8sContext, endpoint string, traceEnabled bool) (*Client, error) { +func dialDirect(ctx context.Context, kubeconfigPath, k8sContext, endpoint string, traceEnabled bool) (*Client, error) { // Always assume TLS to match production behavior creds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) + clientset, err := NewK8sClientset(kubeconfigPath, k8sContext) + if err != nil { + return nil, fmt.Errorf("failed to create k8s client: %w", err) + } + var opts []grpc.DialOption opts = append(opts, grpc.WithTransportCredentials(creds)) opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) + tokenOpt, err := bearerTokenDialOption(ctx, clientset) + if err != nil { + return nil, err + } + opts = append(opts, tokenOpt) if traceEnabled { opts = append(opts, grpc.WithUnaryInterceptor(newTraceInterceptor())) @@ -213,12 +222,12 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext string, tra var opts []grpc.DialOption opts = append(opts, grpc.WithTransportCredentials(transportCreds)) opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) - jwtOpts, err := jwtDialOptions(ctx, clientset) + tokenOpt, err := bearerTokenDialOption(ctx, clientset) if err != nil { close(stopCh) return nil, err } - opts = append(opts, jwtOpts...) + opts = append(opts, tokenOpt) if traceEnabled { opts = append(opts, grpc.WithUnaryInterceptor(newTraceInterceptor())) @@ -241,15 +250,9 @@ func dialPortForward(ctx context.Context, kubeconfigPath, k8sContext string, tra }, nil } -func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grpc.DialOption, error) { - jwtMode, err := isJWTMode(ctx, clientset) - if err != nil { - return nil, err - } - if !jwtMode { - return nil, nil - } - +// bearerTokenDialOption attaches a ServiceAccount token for the ate-client SA +// as per-RPC credentials. +func bearerTokenDialOption(ctx context.Context, clientset *kubernetes.Clientset) (grpc.DialOption, error) { expirationSeconds := int64(3600) tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ @@ -264,38 +267,7 @@ func jwtDialOptions(ctx context.Context, clientset *kubernetes.Clientset) ([]grp if token.Status.Token == "" { return nil, fmt.Errorf("failed to request ateapi bearer token: token response was empty") } - return []grpc.DialOption{grpc.WithPerRPCCredentials(bearerTokenCreds(token.Status.Token))}, nil -} - -func isJWTMode(ctx context.Context, clientset *kubernetes.Clientset) (bool, error) { - // TODO: Replace deployment introspection with an explicit client-readable - // config file once ateapi auth mode is part of install/runtime config. - deployment, err := clientset.AppsV1().Deployments("ate-system").Get(ctx, "ate-api-server", metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("failed to get ate-api-server deployment: %w", err) - } - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name != "ate-api-server" { - continue - } - return isJWTAuthModeArg(container.Args), nil - } - return false, fmt.Errorf("failed to find ate-api-server container in deployment") -} - -func isJWTAuthModeArg(args []string) bool { - for i, arg := range args { - if arg == "--auth-mode=jwt" { - return true - } - if strings.HasPrefix(arg, "--auth-mode=") { - return strings.TrimPrefix(arg, "--auth-mode=") == "jwt" - } - if arg == "--auth-mode" && i+1 < len(args) { - return args[i+1] == "jwt" - } - } - return false + return grpc.WithPerRPCCredentials(bearerTokenCreds(token.Status.Token)), nil } type bearerTokenCreds string diff --git a/internal/ateclient/builder_test.go b/internal/ateclient/builder_test.go index 62c44feb9..3ed7d31a1 100644 --- a/internal/ateclient/builder_test.go +++ b/internal/ateclient/builder_test.go @@ -14,27 +14,25 @@ package ateclient -import "testing" +import ( + "context" + "testing" +) -func TestIsJWTAuthModeArg(t *testing.T) { - tests := []struct { - name string - args []string - want bool - }{ - {name: "equals jwt", args: []string{"--auth-mode=jwt"}, want: true}, - {name: "split jwt", args: []string{"--auth-mode", "jwt"}, want: true}, - {name: "equals mtls", args: []string{"--auth-mode=mtls"}, want: false}, - {name: "split mtls", args: []string{"--auth-mode", "mtls"}, want: false}, - {name: "missing value", args: []string{"--auth-mode"}, want: false}, - {name: "unrelated", args: []string{"--foo=bar"}, want: false}, +func TestBearerTokenCreds(t *testing.T) { + md, err := bearerTokenCreds("some-token").GetRequestMetadata(context.Background()) + if err != nil { + t.Fatalf("GetRequestMetadata: %v", err) + } + if got, want := md["authorization"], "Bearer some-token"; got != want { + t.Errorf("authorization=%q want %q", got, want) + } + + if _, err := bearerTokenCreds("").GetRequestMetadata(context.Background()); err == nil { + t.Error("GetRequestMetadata with empty token: want error, got nil") } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isJWTAuthModeArg(tt.args); got != tt.want { - t.Fatalf("isJWTAuthModeArg(%v) = %v, want %v", tt.args, got, tt.want) - } - }) + if !bearerTokenCreds("some-token").RequireTransportSecurity() { + t.Error("RequireTransportSecurity() = false, want true") } } diff --git a/internal/ateinterceptors/ateinterceptors.go b/internal/ateinterceptors/ateinterceptors.go index 2dfeceaec..55e599248 100644 --- a/internal/ateinterceptors/ateinterceptors.go +++ b/internal/ateinterceptors/ateinterceptors.go @@ -21,6 +21,7 @@ import ( "strconv" "time" + "github.com/agent-substrate/substrate/internal/principal" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -46,12 +47,15 @@ func ServerUnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServer strconv.FormatInt(elapsed.Microseconds(), 10), )) + pInfo, _ := principal.FromContext(ctx) + slog.InfoContext(ctx, "Handle RPC", slog.String("method", info.FullMethod), slog.Any("req", sanitizeForLog(req)), slog.Any("resp", sanitizeForLog(resp)), slog.Any("err", err), slog.String("elapsed-time", elapsed.String()), + slog.Any("principal", pInfo), ) if err != nil { diff --git a/cmd/ateapi/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go similarity index 85% rename from cmd/ateapi/internal/credbundle/credbundle.go rename to internal/credbundle/credbundle.go index 58d86df76..659a53899 100644 --- a/cmd/ateapi/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -38,6 +38,17 @@ func Loader(path string) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { } } +// ClientLoader is the client-side counterpart to Loader. It returns a function +// suitable for use as GetClientCertificate in a tls.Config, re-reading the +// bundle on each handshake so that in-place pod-certificate rotations are +// picked up. +func ClientLoader(path string) func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + // TODO: Introduce caching. + return func(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return Parse(path) + } +} + // Parse reads a private key and certificate chain from a credential bundle file as written by the // Kubernetes Pod Certificates mechanism. func Parse(bundlePath string) (*tls.Certificate, error) { diff --git a/cmd/ateapi/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go similarity index 100% rename from cmd/ateapi/internal/credbundle/credbundle_test.go rename to internal/credbundle/credbundle_test.go diff --git a/internal/principal/principal.go b/internal/principal/principal.go new file mode 100644 index 000000000..c6169aaed --- /dev/null +++ b/internal/principal/principal.go @@ -0,0 +1,45 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package principal + +import "context" + +// Kind values for PrincipalInfo, named after the authentication method that +// established the identity. +const ( + KindMTLS = "mtls" + KindJWT = "jwt" +) + +// PrincipalInfo contains information about an authenticated principal. +type PrincipalInfo struct { + ID string + Kind string +} + +type contextKey struct{} + +var principalKey = contextKey{} + +// FromContext returns the PrincipalInfo from the context, if any. +func FromContext(ctx context.Context) (PrincipalInfo, bool) { + p, ok := ctx.Value(principalKey).(PrincipalInfo) + return p, ok +} + +// InjectContext returns a new context with the given PrincipalInfo. +func InjectContext(ctx context.Context, p PrincipalInfo) context.Context { + return context.WithValue(ctx, principalKey, p) +} diff --git a/internal/substratex509/substratex509.go b/internal/substratex509/substratex509.go new file mode 100644 index 000000000..ee6ee3b40 --- /dev/null +++ b/internal/substratex509/substratex509.go @@ -0,0 +1,131 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package substratex509 contains routines for creating and parsing x509 +// certificates that embed Substrate-specific X.509 extensions communicating +// the identity of a given workload. It is modeled on the upstream Kubernetes +// component-helpers/kubernetesx509 package, but encodes extension values as +// JSON instead of ASN.1. +package substratex509 + +import ( + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "fmt" + "strings" +) + +var ( + // GoogleSubstratePEN is the ASN.1 Private Enterprise Number arc used to + // name X.509 extensions that communicate Substrate-specific concepts. + GoogleSubstratePEN = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 12} + + // oidPodIdentity identifies the Kubernetes PodIdentity X.509 extension specifically in substrate. + oidPodIdentity = makeSubstrateOID(1) +) + +func makeSubstrateOID(subIDs ...int) asn1.ObjectIdentifier { + base := asn1.ObjectIdentifier{} + base = append(base, GoogleSubstratePEN...) + base = append(base, subIDs...) + return base +} + +// PodIdentity is the Kubernetes Pod Identity of a pod, as embedded in the +// oidPodIdentity extension of its certificate. +type PodIdentity struct { + Namespace string + ServiceAccountName string + ServiceAccountUID string + PodName string + PodUID string + NodeName string + NodeUID string +} + +func AddPodIdentityToCertificate(pod *PodIdentity, template *x509.Certificate) error { + if err := validatePodIdentity(pod); err != nil { + return fmt.Errorf("while validating PodIdentity input: %w", err) + } + podIdentityBytes, err := json.Marshal(pod) + if err != nil { + return fmt.Errorf("while json-marshaling PodIdentity extension: %w", err) + } + + template.ExtraExtensions = append(template.ExtraExtensions, pkix.Extension{ + Id: oidPodIdentity, + Value: podIdentityBytes, + }) + + return nil +} + +func PodIdentityFromCertificate(cert *x509.Certificate) (*PodIdentity, error) { + podIdentityCount := 0 + + pod := &PodIdentity{} + for _, ext := range cert.Extensions { + if ext.Id.Equal(oidPodIdentity) { + podIdentityCount++ + + if err := json.Unmarshal(ext.Value, pod); err != nil { + return nil, fmt.Errorf("while json-unmarshaling PodIdentity extension: %w", err) + } + } + } + + if podIdentityCount == 0 { + return nil, nil + } + if podIdentityCount > 1 { + return nil, fmt.Errorf("certificate contains multiple PodIdentity extensions") + } + + if err := validatePodIdentity(pod); err != nil { + return nil, fmt.Errorf("while validating PodIdentity extension: %w", err) + } + + return pod, nil +} + +func validatePodIdentity(pod *PodIdentity) error { + var empty []string + if pod.Namespace == "" { + empty = append(empty, "Namespace") + } + if pod.ServiceAccountName == "" { + empty = append(empty, "ServiceAccountName") + } + if pod.ServiceAccountUID == "" { + empty = append(empty, "ServiceAccountUID") + } + if pod.PodName == "" { + empty = append(empty, "PodName") + } + if pod.PodUID == "" { + empty = append(empty, "PodUID") + } + if pod.NodeName == "" { + empty = append(empty, "NodeName") + } + if pod.NodeUID == "" { + empty = append(empty, "NodeUID") + } + if len(empty) > 0 { + return fmt.Errorf("empty fields: %s", strings.Join(empty, ", ")) + } + return nil +} diff --git a/internal/substratex509/substratex509_test.go b/internal/substratex509/substratex509_test.go new file mode 100644 index 000000000..42aae802e --- /dev/null +++ b/internal/substratex509/substratex509_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package substratex509 + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "math/big" + "reflect" + "strings" + "testing" + "time" +) + +// mintCert self-signs a certificate from the template and parses it back, so +// that ExtraExtensions round-trip into Extensions. +func mintCert(t *testing.T, template *x509.Certificate) *x509.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating key: %v", err) + } + template.SerialNumber = big.NewInt(1) + template.NotBefore = time.Now().Add(-time.Hour) + template.NotAfter = time.Now().Add(time.Hour) + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing certificate: %v", err) + } + return cert +} + +// mintCertWithExtension mints a certificate carrying a single PodIdentity +// extension with the given raw value, bypassing the validation in +// AddPodIdentityToCertificate. +func mintCertWithExtension(t *testing.T, value []byte) *x509.Certificate { + t.Helper() + return mintCert(t, &x509.Certificate{ + ExtraExtensions: []pkix.Extension{ + {Id: testPodIdentityOID, Value: value}, + }, + }) +} + +// testPodIdentityOID spells out the intended PodIdentity extension OID +// to prevent accidental modification to the value. +var testPodIdentityOID = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 12, 1} + +func TestPodIdentityFromCertificate(t *testing.T) { + // fullPodIdentity has every field populated, as required by + // AddPodIdentityToCertificate. Cases that mutate it copy it first. + fullPodIdentity := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + for _, tc := range []struct { + name string + cert func(t *testing.T) *x509.Certificate + want *PodIdentity + wantErr string // substring of the expected error; "" means no error + }{ + { + name: "RoundTrip", + cert: func(t *testing.T) *x509.Certificate { + template := &x509.Certificate{} + if err := AddPodIdentityToCertificate(&fullPodIdentity, template); err != nil { + t.Fatalf("AddPodIdentityToCertificate: %v", err) + } + return mintCert(t, template) + }, + want: &fullPodIdentity, + }, + { + name: "Absent", + cert: func(t *testing.T) *x509.Certificate { + return mintCert(t, &x509.Certificate{}) + }, + want: nil, + }, + { + name: "Duplicate", + cert: func(t *testing.T) *x509.Certificate { + // Go's x509 parser rejects certificates with duplicate + // extensions, so a duplicate can only reach + // PodIdentityFromCertificate via a Certificate constructed + // by other means. Build one directly. + ext := pkix.Extension{Id: testPodIdentityOID, Value: []byte(`{"PodUID":"pod-uid"}`)} + return &x509.Certificate{Extensions: []pkix.Extension{ext, ext}} + }, + wantErr: "multiple PodIdentity extensions", + }, + { + name: "EmptyField", + cert: func(t *testing.T) *x509.Certificate { + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + NodeName: "node-1", + NodeUID: "node-uid", + } + value, err := json.Marshal(pod) + if err != nil { + t.Fatalf("marshaling PodIdentity: %v", err) + } + return mintCertWithExtension(t, value) + }, + wantErr: "PodUID", + }, + { + name: "Malformed", + cert: func(t *testing.T) *x509.Certificate { + return mintCertWithExtension(t, []byte("not json")) + }, + wantErr: "json-unmarshaling", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := PodIdentityFromCertificate(tc.cert(t)) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("PodIdentityFromCertificate succeeded, want error containing %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("PodIdentityFromCertificate error = %q, want it to contain %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("PodIdentityFromCertificate: %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("PodIdentityFromCertificate = %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestAddPodIdentityToCertificateEmptyField(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*PodIdentity) + }{ + {"Namespace", func(p *PodIdentity) { p.Namespace = "" }}, + {"ServiceAccountName", func(p *PodIdentity) { p.ServiceAccountName = "" }}, + {"ServiceAccountUID", func(p *PodIdentity) { p.ServiceAccountUID = "" }}, + {"PodName", func(p *PodIdentity) { p.PodName = "" }}, + {"PodUID", func(p *PodIdentity) { p.PodUID = "" }}, + {"NodeName", func(p *PodIdentity) { p.NodeName = "" }}, + {"NodeUID", func(p *PodIdentity) { p.NodeUID = "" }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + tc.mutate(&pod) + err := AddPodIdentityToCertificate(&pod, &x509.Certificate{}) + if err == nil { + t.Fatalf("AddPodIdentityToCertificate succeeded, want error for empty %s", tc.name) + } + if !strings.Contains(err.Error(), tc.name) { + t.Errorf("error %q does not name the empty field %s", err, tc.name) + } + }) + } +} + +// The extension value is plain JSON so that non-Go verifiers can parse it +// without an ASN.1 library. +func TestExtensionValueIsJSON(t *testing.T) { + pod := PodIdentity{ + Namespace: "ate-system", + ServiceAccountName: "atelet", + ServiceAccountUID: "sa-uid", + PodName: "atelet-abc", + PodUID: "pod-uid", + NodeName: "node-1", + NodeUID: "node-uid", + } + template := &x509.Certificate{} + if err := AddPodIdentityToCertificate(&pod, template); err != nil { + t.Fatalf("AddPodIdentityToCertificate: %v", err) + } + cert := mintCert(t, template) + + for _, ext := range cert.Extensions { + if ext.Id.Equal(testPodIdentityOID) { + if !json.Valid(ext.Value) { + t.Errorf("extension value is not valid JSON: %q", ext.Value) + } + return + } + } + t.Fatal("PodIdentity extension not found in certificate") +} diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index f86294953..5322a661a 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -90,7 +90,8 @@ spec: - --client-jwt-audience=api.ate-system.svc - --session-id-jwt-pool=/run/session-id-jwt-pool/pool.json - --session-id-ca-pool=/run/session-id-ca-pool/pool.json - - --workerpool-ca-certs=/run/workerpool-ca-certs/trust-bundle.pem + - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem env: - name: POD_NAME valueFrom: @@ -117,8 +118,14 @@ spec: name: ate-api-server-envvars optional: true volumeMounts: + # servicedns: the apiserver's own gRPC serving cert (DNS SAN api.ate-system.svc). - name: "servicedns" mountPath: "/run/servicedns.podcert.ate.dev" + # podidentity: the apiserver's client identity (SPIFFE) when it dials + # valkey and atelet, plus the trust bundle for verifying atelet's + # serving certificate. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" - name: "session-id-jwt-pool" mountPath: "/run/session-id-jwt-pool" # Note: See README.md for how to generate this secret. @@ -128,9 +135,6 @@ spec: - name: "session-id-ca-pool" mountPath: "/run/session-id-ca-pool" readOnly: true - - name: "workerpool-ca-certs" - mountPath: "/run/workerpool-ca-certs" - readOnly: true ports: - containerPort: 443 - name: prometheus @@ -149,6 +153,19 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem - name: "session-id-jwt-pool" projected: sources: @@ -173,15 +190,6 @@ spec: items: - key: "pool" path: "pool.json" - - name: "workerpool-ca-certs" - projected: - sources: - - clusterTrustBundle: - signerName: podidentity.podcert.ate.dev/identity - labelSelector: - matchLabels: - podcert.ate.dev/canarying: live - path: trust-bundle.pem --- # 6. Expose the Session Assigner apiVersion: v1 diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 65ee46075..a6955bb81 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -90,6 +90,7 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atecontroller args: - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem ports: - name: metrics containerPort: 8080 @@ -97,3 +98,27 @@ spec: - name: healthz containerPort: 8081 protocol: TCP + volumeMounts: + # Controller's own client identity presented to ateapi. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: "servicedns-ca" + mountPath: "/run/servicedns-ca" + volumes: + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 3ce7f82f1..b8e057a86 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -69,6 +69,8 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atelet args: - --gcp-auth-for-image-pulls=true + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem # atelet does no mounts, netlink, device, or namespace operations (those # live in the ateom worker pod) — it only reads/writes the # /var/lib/ateom-gvisor hostPath as root, so it needs no Linux @@ -118,8 +120,27 @@ spec: volumeMounts: - name: run-ateom mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev volumes: - name: run-ateom hostPath: path: /var/lib/ateom-gvisor type: DirectoryOrCreate + # Identity for mutual TLS with the ate-apiserver. atelet is not behind a + # Service, so it uses a podidentity (SPIFFE) cert rather than a servicedns + # serving cert (which requires DNS SANs it would not have); the + # clusterTrustBundle verifies the apiserver's client certificate. + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 967f22c29..ad6f9dfd6 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -131,12 +131,15 @@ spec: - "--port-xds=18000" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" - - "--ateapi-address=api.ate-system.svc:443" - "--status-port=4040" - "--port-https=8443" - "--envoy-cert-path=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--otlp-collector-address=opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317" + # Client auth to ateapi (mtls): verify the serving cert against the + # servicedns trust bundle and present the podidentity client cert. + - "--ateapi-address=api.ate-system.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" env: - name: POD_NAME valueFrom: @@ -163,6 +166,13 @@ spec: containerPort: 4040 - name: metrics containerPort: 9090 + volumeMounts: + # Router's own client identity presented to ateapi. + - name: "podidentity" + mountPath: "/run/podidentity.podcert.ate.dev" + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: "servicedns-ca" + mountPath: "/run/servicedns-ca" - name: envoy image: envoyproxy/envoy:v1.30-latest command: @@ -194,6 +204,22 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: "podidentity" + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: "servicedns-ca" + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem --- apiVersion: v1 kind: Service diff --git a/manifests/ate-install/kind-jwt/kustomization.yaml b/manifests/ate-install/kind-token-client/kustomization.yaml similarity index 56% rename from manifests/ate-install/kind-jwt/kustomization.yaml rename to manifests/ate-install/kind-token-client/kustomization.yaml index 8769b7a10..d95c6be06 100644 --- a/manifests/ate-install/kind-jwt/kustomization.yaml +++ b/manifests/ate-install/kind-token-client/kustomization.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Overlay: in-cluster clients authenticate to ateapi with a projected +# ServiceAccount token instead of a client certificate. The server always +# accepts both. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -19,17 +22,11 @@ resources: - ../kind patches: - - path: ../jwt/patches.yaml - - target: - group: apps - version: v1 - kind: Deployment - name: ate-api-server - namespace: ate-system - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --auth-mode=jwt + - path: ../token-client/patches.yaml + # --ateapi-use-token-auth switches the client to the projected + # ServiceAccount token below. The base manifests' --ateapi-client-cert is + # removed by index; the preceding `test` op pins the expected value so the + # build fails loudly if the base args are ever reordered. - target: group: apps version: v1 @@ -37,9 +34,14 @@ patches: name: ate-controller namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/1 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/1 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token @@ -50,9 +52,14 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/13 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 8463bfd79..6b28ff2fd 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -33,6 +33,8 @@ patches: args: - --gcp-auth-for-image-pulls=false - --localhost-registry-replacement=kind-registry:5000 + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: http://opentelemetry-collector.otel-system.svc:4317 diff --git a/manifests/ate-install/jwt/kustomization.yaml b/manifests/ate-install/token-client/kustomization.yaml similarity index 60% rename from manifests/ate-install/jwt/kustomization.yaml rename to manifests/ate-install/token-client/kustomization.yaml index a6331c498..82e39fc7f 100644 --- a/manifests/ate-install/jwt/kustomization.yaml +++ b/manifests/ate-install/token-client/kustomization.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Overlay: in-cluster clients authenticate to ateapi with a projected +# ServiceAccount token instead of a client certificate. The server always +# accepts both. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -26,16 +29,10 @@ resources: patches: - path: patches.yaml - - target: - group: apps - version: v1 - kind: Deployment - name: ate-api-server - namespace: ate-system - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --auth-mode=jwt + # --ateapi-use-token-auth switches the client to the projected + # ServiceAccount token below. The base manifests' --ateapi-client-cert is + # removed by index; the preceding `test` op pins the expected value so the + # build fails loudly if the base args are ever reordered. - target: group: apps version: v1 @@ -43,9 +40,14 @@ patches: name: ate-controller namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/1 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/1 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token @@ -56,9 +58,14 @@ patches: name: atenet-router namespace: ate-system patch: |- + - op: test + path: /spec/template/spec/containers/0/args/13 + value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - op: remove + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- - value: --ateapi-auth=jwt + value: --ateapi-use-token-auth=true - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-token-file=/run/ateapi-token/token diff --git a/manifests/ate-install/jwt/patches.yaml b/manifests/ate-install/token-client/patches.yaml similarity index 100% rename from manifests/ate-install/jwt/patches.yaml rename to manifests/ate-install/token-client/patches.yaml diff --git a/manifests/ate-install/valkey.yaml b/manifests/ate-install/valkey.yaml index 2e5fb0088..435352106 100644 --- a/manifests/ate-install/valkey.yaml +++ b/manifests/ate-install/valkey.yaml @@ -28,6 +28,8 @@ data: # Load certificates from projected volume tls-cert-file /run/servicedns.podcert.ate.dev/credential-bundle.pem tls-key-file /run/servicedns.podcert.ate.dev/credential-bundle.pem + tls-client-cert-file /run/podidentity.podcert.ate.dev/credential-bundle.pem + tls-client-key-file /run/podidentity.podcert.ate.dev/credential-bundle.pem tls-ca-cert-file /etc/valkey-ca/ca.crt tls-auth-clients yes # Reload the cert every 12h. @@ -114,6 +116,8 @@ spec: mountPath: /etc/valkey - name: servicedns mountPath: /run/servicedns.podcert.ate.dev + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev - name: valkey-ca-certs mountPath: /etc/valkey-ca readOnly: true @@ -130,6 +134,13 @@ spec: signerName: servicedns.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem - name: valkey-ca-certs projected: sources: @@ -163,8 +174,11 @@ spec: - name: init image: valkey/valkey:9.1@sha256:4963247afc4cd33c7d3b2d2816b9f7f8eeebab148d29056c2ca4d7cbc966f2d9 volumeMounts: - - name: servicedns - mountPath: /run/servicedns.podcert.ate.dev + # The init job is a client of the cluster, not a Service-backed server, + # so it uses a podidentity client cert rather than a servicedns + # serving cert (which requires DNS SANs it would not have). + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev - name: valkey-ca-certs mountPath: /etc/valkey-ca readOnly: true @@ -188,19 +202,19 @@ spec: done echo "Checking if Valkey cluster is already initialized..." - until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc ping >/dev/null 2>&1; do + until valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc ping >/dev/null 2>&1; do echo "Waiting for valkey-cluster-0 to respond to ping..." sleep 2 done - INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem --key /run/servicedns.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc cluster info 2>/dev/null | grep cluster_state || true) + INIT_STATUS=$(valkey-cli --tls --cacert /etc/valkey-ca/ca.crt --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem --key /run/podidentity.podcert.ate.dev/credential-bundle.pem -h valkey-cluster-0.valkey-cluster-service.ate-system.svc cluster info 2>/dev/null | grep cluster_state || true) if [ -z "${INIT_STATUS}" ] || ! echo "${INIT_STATUS}" | grep -q "cluster_state:ok"; then echo "Initializing Valkey cluster..." valkey-cli --tls \ --cacert /etc/valkey-ca/ca.crt \ - --cert /run/servicedns.podcert.ate.dev/credential-bundle.pem \ - --key /run/servicedns.podcert.ate.dev/credential-bundle.pem \ + --cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ --cluster create ${VALKEY_NODES} \ --cluster-replicas 1 \ --cluster-yes @@ -209,11 +223,11 @@ spec: echo "Cluster already initialized." fi volumes: - - name: servicedns + - name: podidentity projected: sources: - podCertificate: - signerName: servicedns.podcert.ate.dev/identity + signerName: podidentity.podcert.ate.dev/identity keyType: ECDSAP256 credentialBundlePath: credential-bundle.pem - name: valkey-ca-certs diff --git a/vendor/google.golang.org/grpc/health/client.go b/vendor/google.golang.org/grpc/health/client.go new file mode 100644 index 000000000..740745c45 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/client.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "fmt" + "io" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/status" +) + +var ( + backoffStrategy = backoff.DefaultExponential + backoffFunc = func(ctx context.Context, retries int) bool { + d := backoffStrategy.Backoff(retries) + timer := time.NewTimer(d) + select { + case <-timer.C: + return true + case <-ctx.Done(): + timer.Stop() + return false + } + } +) + +func init() { + internal.HealthCheckFunc = clientHealthCheck +} + +const healthCheckMethod = "/grpc.health.v1.Health/Watch" + +// This function implements the protocol defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +func clientHealthCheck(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), service string) error { + tryCnt := 0 + +retryConnection: + for { + // Backs off if the connection has failed in some way without receiving a message in the previous retry. + if tryCnt > 0 && !backoffFunc(ctx, tryCnt-1) { + return nil + } + tryCnt++ + + if ctx.Err() != nil { + return nil + } + setConnectivityState(connectivity.Connecting, nil) + rawS, err := newStream(healthCheckMethod) + if err != nil { + continue retryConnection + } + + s, ok := rawS.(grpc.ClientStream) + // Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes. + if !ok { + setConnectivityState(connectivity.Ready, nil) + return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS) + } + + if err = s.SendMsg(&healthpb.HealthCheckRequest{Service: service}); err != nil && err != io.EOF { + // Stream should have been closed, so we can safely continue to create a new stream. + continue retryConnection + } + s.CloseSend() + + resp := new(healthpb.HealthCheckResponse) + for { + err = s.RecvMsg(resp) + + // Reports healthy for the LBing purposes if health check is not implemented in the server. + if status.Code(err) == codes.Unimplemented { + setConnectivityState(connectivity.Ready, nil) + return err + } + + // Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED. + if err != nil { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but received health check RPC error: %v", err)) + continue retryConnection + } + + // As a message has been received, removes the need for backoff for the next retry by resetting the try count. + tryCnt = 0 + if resp.Status == healthpb.HealthCheckResponse_SERVING { + setConnectivityState(connectivity.Ready, nil) + } else { + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but health check failed. status=%s", resp.Status)) + } + } + } +} diff --git a/vendor/google.golang.org/grpc/health/logging.go b/vendor/google.golang.org/grpc/health/logging.go new file mode 100644 index 000000000..83c6acf55 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/logging.go @@ -0,0 +1,23 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import "google.golang.org/grpc/grpclog" + +var logger = grpclog.Component("health_service") diff --git a/vendor/google.golang.org/grpc/health/producer.go b/vendor/google.golang.org/grpc/health/producer.go new file mode 100644 index 000000000..f938e5790 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/producer.go @@ -0,0 +1,106 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/status" +) + +func init() { + producerBuilderSingleton = &producerBuilder{} + internal.RegisterClientHealthCheckListener = registerClientSideHealthCheckListener +} + +type producerBuilder struct{} + +var producerBuilderSingleton *producerBuilder + +// Build constructs and returns a producer and its cleanup function. +func (*producerBuilder) Build(cci any) (balancer.Producer, func()) { + p := &healthServiceProducer{ + cc: cci.(grpc.ClientConnInterface), + cancel: func() {}, + } + return p, func() { + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + } +} + +type healthServiceProducer struct { + // The following fields are initialized at build time and read-only after + // that and therefore do not need to be guarded by a mutex. + cc grpc.ClientConnInterface + + mu sync.Mutex + cancel func() +} + +// registerClientSideHealthCheckListener accepts a listener to provide server +// health state via the health service. +func registerClientSideHealthCheckListener(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) func() { + pr, closeFn := sc.GetOrBuildProducer(producerBuilderSingleton) + p := pr.(*healthServiceProducer) + p.mu.Lock() + defer p.mu.Unlock() + p.cancel() + if listener == nil { + return closeFn + } + + ctx, cancel := context.WithCancel(ctx) + p.cancel = cancel + + go p.startHealthCheck(ctx, sc, serviceName, listener) + return closeFn +} + +func (p *healthServiceProducer) startHealthCheck(ctx context.Context, sc balancer.SubConn, serviceName string, listener func(balancer.SubConnState)) { + newStream := func(method string) (any, error) { + return p.cc.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, method) + } + + setConnectivityState := func(state connectivity.State, err error) { + listener(balancer.SubConnState{ + ConnectivityState: state, + ConnectionError: err, + }) + } + + // Call the function through the internal variable as tests use it for + // mocking. + err := internal.HealthCheckFunc(ctx, newStream, setConnectivityState, serviceName) + if err == nil { + return + } + if status.Code(err) == codes.Unimplemented { + logger.Errorf("Subchannel health check is unimplemented at server side, thus health check is disabled for SubConn %p", sc) + } else { + logger.Errorf("Health checking failed for SubConn %p: %v", sc, err) + } +} diff --git a/vendor/google.golang.org/grpc/health/server.go b/vendor/google.golang.org/grpc/health/server.go new file mode 100644 index 000000000..d8eebe99d --- /dev/null +++ b/vendor/google.golang.org/grpc/health/server.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package health provides a service that exposes server's health and it must be +// imported to enable support for client-side health checks. +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc/codes" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +const ( + // maxAllowedServices defines the maximum number of resources a List + // operation can return. An error is returned if the number of services + // exceeds this limit. + maxAllowedServices = 100 +) + +// Server implements `service Health`. +type Server struct { + healthgrpc.UnimplementedHealthServer + mu sync.RWMutex + // If shutdown is true, it's expected all serving status is NOT_SERVING, and + // will stay in NOT_SERVING. + shutdown bool + // statusMap stores the serving status of the services this Server monitors. + statusMap map[string]healthpb.HealthCheckResponse_ServingStatus + updates map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus +} + +// NewServer returns a new Server. +func NewServer() *Server { + return &Server{ + statusMap: map[string]healthpb.HealthCheckResponse_ServingStatus{"": healthpb.HealthCheckResponse_SERVING}, + updates: make(map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus), + } +} + +// Check implements `service Health`. +func (s *Server) Check(_ context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if servingStatus, ok := s.statusMap[in.Service]; ok { + return &healthpb.HealthCheckResponse{ + Status: servingStatus, + }, nil + } + return nil, status.Error(codes.NotFound, "unknown service") +} + +// List implements `service Health`. +func (s *Server) List(_ context.Context, _ *healthpb.HealthListRequest) (*healthpb.HealthListResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if len(s.statusMap) > maxAllowedServices { + return nil, status.Errorf(codes.ResourceExhausted, "server health list exceeds maximum capacity: %d", maxAllowedServices) + } + + statusMap := make(map[string]*healthpb.HealthCheckResponse, len(s.statusMap)) + for k, v := range s.statusMap { + statusMap[k] = &healthpb.HealthCheckResponse{Status: v} + } + + return &healthpb.HealthListResponse{Statuses: statusMap}, nil +} + +// Watch implements `service Health`. +func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + service := in.Service + // update channel is used for getting service status updates. + update := make(chan healthpb.HealthCheckResponse_ServingStatus, 1) + s.mu.Lock() + // Puts the initial status to the channel. + if servingStatus, ok := s.statusMap[service]; ok { + update <- servingStatus + } else { + update <- healthpb.HealthCheckResponse_SERVICE_UNKNOWN + } + + // Registers the update channel to the correct place in the updates map. + if _, ok := s.updates[service]; !ok { + s.updates[service] = make(map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus) + } + s.updates[service][stream] = update + defer func() { + s.mu.Lock() + delete(s.updates[service], stream) + s.mu.Unlock() + }() + s.mu.Unlock() + + var lastSentStatus healthpb.HealthCheckResponse_ServingStatus = -1 + for { + select { + // Status updated. Sends the up-to-date status to the client. + case servingStatus := <-update: + if lastSentStatus == servingStatus { + continue + } + lastSentStatus = servingStatus + err := stream.Send(&healthpb.HealthCheckResponse{Status: servingStatus}) + if err != nil { + return status.Error(codes.Canceled, "Stream has ended.") + } + // Context done. Removes the update channel from the updates map. + case <-stream.Context().Done(): + return status.Error(codes.Canceled, "Stream has ended.") + } + } +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + defer s.mu.Unlock() + if s.shutdown { + logger.Infof("health: status changing for %s to %v is ignored because health service is shutdown", service, servingStatus) + return + } + + s.setServingStatusLocked(service, servingStatus) +} + +func (s *Server) setServingStatusLocked(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.statusMap[service] = servingStatus + for _, update := range s.updates[service] { + // Clears previous updates, that are not sent to the client, from the channel. + // This can happen if the client is not reading and the server gets flow control limited. + select { + case <-update: + default: + } + // Puts the most recent update to the channel. + update <- servingStatus + } +} + +// Shutdown sets all serving status to NOT_SERVING, and configures the server to +// ignore all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Shutdown() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = true + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_NOT_SERVING) + } +} + +// Resume sets all serving status to SERVING, and configures the server to +// accept all future status changes. +// +// This changes serving status for all services. To set status for a particular +// services, call SetServingStatus(). +func (s *Server) Resume() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = false + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_SERVING) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 13877b16e..5a18bf468 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1031,6 +1031,7 @@ google.golang.org/grpc/experimental/opentelemetry google.golang.org/grpc/experimental/stats google.golang.org/grpc/grpclog google.golang.org/grpc/grpclog/internal +google.golang.org/grpc/health google.golang.org/grpc/health/grpc_health_v1 google.golang.org/grpc/internal google.golang.org/grpc/internal/admin