Added mTLS between ate system components

.github/workflows/pr-workflow.yaml

@@ -43,8 +43,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        auth-mode: [mtls, jwt]
-    name: e2e-test (${{ matrix.auth-mode }})
+        ateapi-client-auth: [cert, token]
+    name: e2e-test (${{ matrix.ateapi-client-auth }})
     steps:
     - name: Checkout
       uses: actions/checkout@v5
@@ -88,11 +88,11 @@ jobs:
     - name: Create cluster
       run: hack/create-kind-cluster.sh
     - name: Install Agent Substrate
-      run: hack/install-ate-kind.sh --deploy-ate-system --auth-mode=${{ matrix.auth-mode }}
+      run: hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth=${{ matrix.ateapi-client-auth }}
     - name: Deploy micro-VM counter demo
       # Stages the (cached) assets into the cluster's rustfs and applies the
       # counter-microvm demo onto the control plane installed above.
-      run: hack/run-microvm-demo-kind.sh --auth-mode=${{ matrix.auth-mode }}
+      run: hack/run-microvm-demo-kind.sh --ateapi-client-auth=${{ matrix.ateapi-client-auth }}
     - name: Deploy gVisor counter demo
       run: hack/install-ate-kind.sh --deploy-demo-counter
     - name: Wait for micro-VM golden snapshot

cmd/ateapi/internal/controlapi/dialer.go

@@ -15,12 +15,18 @@
 package controlapi
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 
+	"os"
+
+	"github.com/agent-substrate/substrate/internal/credbundle"
+	"github.com/agent-substrate/substrate/internal/substratex509"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/utils/lru"
@@ -33,14 +39,26 @@ type AteletDialer struct {
 	workerIndexer cache.Indexer
 	ateletIndexer cache.Indexer
 	ateletConns   *lru.Cache
+	// dialCredentials builds the transport credentials used to dial a given
+	// atelet, keyed on the atelet's expected pod UID. Production wires this to
+	// per-atelet mTLS; tests can override it with insecure credentials.
+	dialCredentials func(expectedPodUID string) (credentials.TransportCredentials, error)
 }
 
-// NewAteletDialer creates a new AteletDialer.
-func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer) *AteletDialer {
+// NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath
+// are used to build the per-atelet mTLS credentials used for every atelet connection.
+func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string) *AteletDialer {
 	return &AteletDialer{
 		workerIndexer: workerIndexer,
 		ateletIndexer: ateletIndexer,
 		ateletConns:   lru.New(1024),
+		dialCredentials: func(expectedPodUID string) (credentials.TransportCredentials, error) {
+			tlsConfig, err := buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID)
+			if err != nil {
+				return nil, err
+			}
+			return credentials.NewTLS(tlsConfig), nil
+		},
 	}
 }
 
@@ -73,20 +91,25 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) (
 	}
 
 	selectedAtelet := matchingAtelets[0].(*corev1.Pod)
-	ateletKey := selectedAtelet.ObjectMeta.Namespace + "/" + selectedAtelet.ObjectMeta.Name
+	ateletKey := string(selectedAtelet.ObjectMeta.UID)
 
 	ateletConnAny, ok := d.ateletConns.Get(ateletKey)
 	if ok {
 		return ateletConnAny.(*grpc.ClientConn), nil
 	}
 
 	if len(selectedAtelet.Status.PodIPs) == 0 {
-		return nil, fmt.Errorf("selected atelet %q has no assigned IPs: %w", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name, err)
+		return nil, fmt.Errorf("selected atelet %q has no assigned IPs", selectedAtelet.ObjectMeta.Namespace+"/"+selectedAtelet.ObjectMeta.Name)
+	}
+
+	creds, err := d.dialCredentials(string(selectedAtelet.ObjectMeta.UID))
+	if err != nil {
+		return nil, fmt.Errorf("while building atelet credentials: %w", err)
 	}
 
 	ateletConn, err := grpc.NewClient(
 		selectedAtelet.Status.PodIPs[0].IP+":8085",
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithTransportCredentials(creds),
 		grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
 	)
 	if err != nil {
@@ -97,3 +120,70 @@ func (d *AteletDialer) DialForWorker(workerPodNamespace, workerPodName string) (
 
 	return ateletConn, nil
 }
+
+func buildTLSConfig(clientBundlePath, serverCAPath, expectedPodUID string) (*tls.Config, error) {
+	roots, err := caPoolFromFile(serverCAPath)
+	if err != nil {
+		return nil, err
+	}
+
+	tlsConfig := tls.Config{
+		MinVersion:           tls.VersionTLS13,
+		GetClientCertificate: credbundle.ClientLoader(clientBundlePath),
+		// Skip the default verification because the peer is dialed by IP and its
+		// certificate has no DNS/IP SAN.
+		InsecureSkipVerify: true,
+		RootCAs:            roots,
+		VerifyConnection:   verifyAteletServerCert(roots, expectedPodUID),
+	}
+
+	return &tlsConfig, nil
+}
+
+func verifyAteletServerCert(roots *x509.CertPool, expectedPodUID string) func(tls.ConnectionState) error {
+	return func(cs tls.ConnectionState) error {
+		if len(cs.PeerCertificates) == 0 {
+			return fmt.Errorf("server presented no certificate")
+		}
+		leaf := cs.PeerCertificates[0]
+		intermediates := x509.NewCertPool()
+		for _, c := range cs.PeerCertificates[1:] {
+			intermediates.AddCert(c)
+		}
+		if _, err := leaf.Verify(x509.VerifyOptions{
+			Roots:         roots,
+			Intermediates: intermediates,
+			KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		}); err != nil {
+			return fmt.Errorf("verifying server certificate chain: %w", err)
+		}
+
+		identity, err := substratex509.PodIdentityFromCertificate(leaf)
+		if err != nil {
+			return fmt.Errorf("failed to parse PodIdentity extension: %w", err)
+		}
+		if identity == nil {
+			return fmt.Errorf("server certificate has no PodIdentity extension, expected pod UID %q", expectedPodUID)
+		}
+		if identity.PodUID == "" || expectedPodUID == "" {
+			return fmt.Errorf("found pod UID == %q, expected pod UID == %q", identity.PodUID, expectedPodUID)
+		}
+		if identity.PodUID != expectedPodUID {
+			return fmt.Errorf("pod UID %q does not match expected %q", identity.PodUID, expectedPodUID)
+		}
+
+		return nil
+	}
+}
+
+func caPoolFromFile(path string) (*x509.CertPool, error) {
+	caBytes, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CA bundle from %s: %w", path, err)
+	}
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(caBytes) {
+		return nil, fmt.Errorf("failed to parse CA bundle from %s", path)
+	}
+	return pool, nil
+}

cmd/ateapi/internal/controlapi/dialer_test.go

@@ -0,0 +1,160 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlapi
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/agent-substrate/substrate/internal/substratex509"
+)
+
+// makeTestCA mints a self-signed CA and returns it along with a pool
+// containing it as the sole root.
+func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509.CertPool) {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generating CA key: %v", err)
+	}
+	template := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "test-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	if err != nil {
+		t.Fatalf("creating CA certificate: %v", err)
+	}
+	cert, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parsing CA certificate: %v", err)
+	}
+	pool := x509.NewCertPool()
+	pool.AddCert(cert)
+	return cert, key, pool
+}
+
+// makeLeafCert mints a server leaf certificate signed by the given CA. If
+// podUID is non-empty it is embedded in a PodIdentity extension.
+func makeLeafCert(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, podUID string) *x509.Certificate {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generating leaf key: %v", err)
+	}
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(2),
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+	if podUID != "" {
+		// AddPodIdentityToCertificate requires all fields to be non-empty;
+		// only PodUID matters to these tests.
+		err := substratex509.AddPodIdentityToCertificate(&substratex509.PodIdentity{
+			Namespace:          "ate-system",
+			ServiceAccountName: "atelet",
+			ServiceAccountUID:  "sa-uid",
+			PodName:            "atelet-abc",
+			PodUID:             podUID,
+			NodeName:           "node-1",
+			NodeUID:            "node-uid",
+		}, template)
+		if err != nil {
+			t.Fatalf("adding PodIdentity extension: %v", err)
+		}
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, ca, &key.PublicKey, caKey)
+	if err != nil {
+		t.Fatalf("creating leaf certificate: %v", err)
+	}
+	cert, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parsing leaf certificate: %v", err)
+	}
+	return cert
+}
+
+func TestVerifyAteletServerCert(t *testing.T) {
+	ca, caKey, roots := makeTestCA(t)
+	otherCA, otherCAKey, _ := makeTestCA(t)
+
+	const uid = "5a2e1c9f-0b57-4a52-9f6e-2f6d3a1b8c4d"
+
+	tests := []struct {
+		name        string
+		leaf        *x509.Certificate
+		expectedUID string
+		wantErr     bool
+	}{
+		{
+			name:        "matching UID succeeds",
+			leaf:        makeLeafCert(t, ca, caKey, uid),
+			expectedUID: uid,
+		},
+		{
+			name:        "mismatched UID fails",
+			leaf:        makeLeafCert(t, ca, caKey, "some-other-uid"),
+			expectedUID: uid,
+			wantErr:     true,
+		},
+		{
+			name:        "missing pod UID extension fails",
+			leaf:        makeLeafCert(t, ca, caKey, ""),
+			expectedUID: uid,
+			wantErr:     true,
+		},
+		{
+			name:        "cert from untrusted CA fails",
+			leaf:        makeLeafCert(t, otherCA, otherCAKey, uid),
+			expectedUID: uid,
+			wantErr:     true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			verify := verifyAteletServerCert(roots, tc.expectedUID)
+			err := verify(tls.ConnectionState{
+				PeerCertificates: []*x509.Certificate{tc.leaf},
+			})
+			if gotErr := err != nil; gotErr != tc.wantErr {
+				t.Fatalf("verify returned error %v, wantErr=%v", err, tc.wantErr)
+			}
+		})
+	}
+
+	t.Run("no peer certificate fails", func(t *testing.T) {
+		verify := verifyAteletServerCert(roots, uid)
+		if err := verify(tls.ConnectionState{}); err == nil {
+			t.Fatal("verify succeeded, want error")
+		}
+	})
+}

cmd/ateapi/internal/controlapi/functional_test.go

@@ -42,6 +42,7 @@ import (
 	"github.com/redis/go-redis/v9"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
@@ -309,7 +310,13 @@ func setupTest(t *testing.T, ns string) *testContext {
 		t.Fatalf("failed to start worker cache: %v", err)
 	}
 
-	dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer())
+	dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "")
+	// Dial the fake atelet over insecure transport instead of per-atelet mTLS,
+	// so DialForWorker's real lookup/dial/cache path is exercised under test.
+	dialer.dialCredentials = func(_ string) (credentials.TransportCredentials, error) {
+		return insecure.NewCredentials(), nil
+	}
+
 	service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient)
 
 	// 5. Start REAL gRPC Server for ATE API