fix(icms-translate): run LLM sidecars as a shared UID so pylon can read the worker-token

src/compute-plane-services/nvca/internal/miniservice/translate_workload.go

@@ -89,6 +89,7 @@ func (r *Reconciler) translateFunctionWorkload(
 		TranslateConfig:        cmnTCfg,
 		DefaultStargateAddress: r.cfg.Workload.DefaultStargateAddress,
 		StargateQUICInsecure:   r.cfg.Workload.StargateQUICInsecure,
+		LLMSidecarRunAsUser:    r.cfg.Workload.LLMSidecarRunAsUser,
 	}
 
 	msg := function.CreationQueueMessage{

src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go

@@ -285,6 +285,7 @@ func (c K8sComputeBackend) translateFunctionLaunchSpecification(
 		},
 		DefaultStargateAddress: c.bk8s.cfg.Workload.DefaultStargateAddress,
 		StargateQUICInsecure:   c.bk8s.cfg.Workload.StargateQUICInsecure,
+		LLMSidecarRunAsUser:    c.bk8s.cfg.Workload.LLMSidecarRunAsUser,
 	}
 
 	if reqType == ftContainer {

src/compute-plane-services/nvca/pkg/storage/translate_workload.go

@@ -85,6 +85,7 @@ func (r *Reconciler) translateFunctionWorkload(
 		TranslateConfig:        cmnTCfg,
 		DefaultStargateAddress: r.cfg.Workload.DefaultStargateAddress,
 		StargateQUICInsecure:   r.cfg.Workload.StargateQUICInsecure,
+		LLMSidecarRunAsUser:    r.cfg.Workload.LLMSidecarRunAsUser,
 	}
 
 	msg := function.CreationQueueMessage{

src/libraries/go/lib/pkg/icms-translate/translate/function/llm.go

@@ -40,8 +40,53 @@ const (
 
 	llmDirMountPath    = "/var/run/llm"
 	llmWorkerTokenPath = llmDirMountPath + "/worker-token"
+
+	// pylon's image user (nvs); both LLM sidecars share it so pylon can read
+	// the credential-manager's 0600 worker-token. Override via
+	// TranslateConfig.LLMSidecarRunAsUser (negative = unset for OpenShift SCC).
+	defaultLLMSidecarRunAsUser int64 = 1000
 )
 
+// resolveLLMSidecarRunAsUser returns the shared sidecar UID, or nil to leave
+// it unset for the platform to assign.
+func resolveLLMSidecarRunAsUser(tcfg TranslateConfig) *int64 {
+	if tcfg.LLMSidecarRunAsUser == nil {
+		uid := defaultLLMSidecarRunAsUser
+		return &uid
+	}
+	if *tcfg.LLMSidecarRunAsUser < 0 {
+		return nil
+	}
+	return tcfg.LLMSidecarRunAsUser
+}
+
+// newLLMSidecarSecurityContext runs both LLM sidecars as the same UID so they
+// can exchange the worker-token on the shared emptyDir.
+func newLLMSidecarSecurityContext(tcfg TranslateConfig) *corev1.SecurityContext {
+	uid := resolveLLMSidecarRunAsUser(tcfg)
+	if uid == nil {
+		return nil
+	}
+	return &corev1.SecurityContext{
+		RunAsUser:  uid,
+		RunAsGroup: uid,
+	}
+}
+
+// applyLLMPodSecurityContext sets the pod fsGroup to the sidecar UID so the
+// non-root sidecars can write the shared worker-token emptyDir. No-op when the
+// UID is platform-managed.
+func applyLLMPodSecurityContext(pod *corev1.Pod, tcfg TranslateConfig) {
+	uid := resolveLLMSidecarRunAsUser(tcfg)
+	if uid == nil {
+		return
+	}
+	if pod.Spec.SecurityContext == nil {
+		pod.Spec.SecurityContext = &corev1.PodSecurityContext{}
+	}
+	pod.Spec.SecurityContext.FSGroup = uid
+}
+
 func normalizeLLMRequestRouterAddressEnvAliases(envSet map[string]string) {
 	llmRequestRouterAddress := envSet[llmRequestRouterAddressEnv]
 	if llmRequestRouterAddress == "" {
@@ -143,6 +188,7 @@ func newLLMRouterClientContainer(
 		Name:            LLMWorkerContainerName,
 		Image:           llmRouterClientImage,
 		ImagePullPolicy: corev1.PullIfNotPresent,
+		SecurityContext: newLLMSidecarSecurityContext(tcfg),
 		Args:            args,
 		Env:             common.SortEnvs(envs),
 		Resources: corev1.ResourceRequirements{
@@ -174,7 +220,7 @@ func newLLMRouterClientContainer(
 	return c, nil
 }
 
-func newLLMCredentialManagerContainer(allEnvSet map[string]string, _ TranslateConfig) (corev1.Container, error) {
+func newLLMCredentialManagerContainer(allEnvSet map[string]string, tcfg TranslateConfig) (corev1.Container, error) {
 	llmCredentialManagerImage := allEnvSet[llmCredentialManagerImageEnv]
 	if llmCredentialManagerImage == "" {
 		llmCredentialManagerImage = llmCredentialManagerImageDefault
@@ -217,6 +263,7 @@ func newLLMCredentialManagerContainer(allEnvSet map[string]string, _ TranslateCo
 		Name:            "llm-credential-manager",
 		Image:           llmCredentialManagerImage,
 		ImagePullPolicy: corev1.PullIfNotPresent,
+		SecurityContext: newLLMSidecarSecurityContext(tcfg),
 		Env:             common.SortEnvs(envs),
 		Resources: corev1.ResourceRequirements{
 			Requests: corev1.ResourceList{

src/libraries/go/lib/pkg/icms-translate/translate/function/llm_test.go

@@ -431,3 +431,67 @@ func envSliceToMap(envs []corev1.EnvVar) map[string]string {
 	}
 	return m
 }
+
+// TestLLMSidecarsShareRunAsUser guards the fix for the worker-token permission
+// bug: the credential-manager writes the token 0600, so it and pylon must run
+// as the same UID for pylon to read it. The UID is configurable for platforms
+// (e.g. OpenShift) whose SCC stipulates the allowed range.
+func TestLLMSidecarsShareRunAsUser(t *testing.T) {
+	allEnvSet := map[string]string{
+		"LLM_REQUEST_ROUTER_ADDRESS": "llm-router.example.com:443",
+		"INFERENCE_PORT":             "8080",
+	}
+	ptr := func(v int64) *int64 { return &v }
+	build := func(t *testing.T, tcfg TranslateConfig) (corev1.Container, corev1.Container) {
+		t.Helper()
+		pylon, err := newLLMRouterClientContainer(&LaunchSpecification{}, allEnvSet, tcfg, "inst-123", false)
+		require.NoError(t, err)
+		cred, err := newLLMCredentialManagerContainer(allEnvSet, tcfg)
+		require.NoError(t, err)
+		return pylon, cred
+	}
+
+	t.Run("default is the shared non-root pylon UID", func(t *testing.T) {
+		pylon, cred := build(t, TranslateConfig{})
+		for name, sc := range map[string]*corev1.SecurityContext{"pylon": pylon.SecurityContext, "credential-manager": cred.SecurityContext} {
+			require.NotNilf(t, sc, "%s security context", name)
+			require.NotNilf(t, sc.RunAsUser, "%s runAsUser", name)
+			assert.Equalf(t, defaultLLMSidecarRunAsUser, *sc.RunAsUser, "%s runAsUser", name)
+			require.NotNilf(t, sc.RunAsGroup, "%s runAsGroup", name)
+			assert.Equalf(t, defaultLLMSidecarRunAsUser, *sc.RunAsGroup, "%s runAsGroup", name)
+			assert.NotZerof(t, *sc.RunAsUser, "%s must be non-root", name)
+		}
+	})
+
+	t.Run("configured UID overrides the default on both sidecars", func(t *testing.T) {
+		pylon, cred := build(t, TranslateConfig{LLMSidecarRunAsUser: ptr(2000)})
+		require.NotNil(t, pylon.SecurityContext.RunAsUser)
+		require.NotNil(t, cred.SecurityContext.RunAsUser)
+		assert.Equal(t, int64(2000), *pylon.SecurityContext.RunAsUser)
+		assert.Equal(t, *pylon.SecurityContext.RunAsUser, *cred.SecurityContext.RunAsUser)
+	})
+
+	t.Run("negative UID leaves runAsUser unset for the platform", func(t *testing.T) {
+		pylon, cred := build(t, TranslateConfig{LLMSidecarRunAsUser: ptr(-1)})
+		assert.Nil(t, pylon.SecurityContext)
+		assert.Nil(t, cred.SecurityContext)
+	})
+}
+
+func TestApplyLLMPodSecurityContext(t *testing.T) {
+	ptr := func(v int64) *int64 { return &v }
+
+	t.Run("sets fsGroup to the sidecar UID", func(t *testing.T) {
+		pod := &corev1.Pod{}
+		applyLLMPodSecurityContext(pod, TranslateConfig{})
+		require.NotNil(t, pod.Spec.SecurityContext)
+		require.NotNil(t, pod.Spec.SecurityContext.FSGroup)
+		assert.Equal(t, defaultLLMSidecarRunAsUser, *pod.Spec.SecurityContext.FSGroup)
+	})
+
+	t.Run("negative UID leaves the pod security context unset", func(t *testing.T) {
+		pod := &corev1.Pod{}
+		applyLLMPodSecurityContext(pod, TranslateConfig{LLMSidecarRunAsUser: ptr(-1)})
+		assert.Nil(t, pod.Spec.SecurityContext)
+	})
+}

src/libraries/go/lib/pkg/icms-translate/translate/function/translate.go

@@ -46,6 +46,10 @@ type TranslateConfig struct {
 	// Whether to use QUIC insecure mode for stargate connections
 	// If not set, "--quick-insecure" is omitted
 	StargateQUICInsecure bool `json:"stargateQUICInsecure,omitempty"`
+
+	// LLMSidecarRunAsUser is the UID both LLM sidecars run as (default 1000).
+	// Negative leaves runAsUser unset so the platform (e.g. OpenShift) assigns.
+	LLMSidecarRunAsUser *int64 `json:"llmSidecarRunAsUser,omitempty"`
 }
 
 func Translate(t CreationQueueMessage, tcfg TranslateConfig) (objs []metav1.Object, err error) {

src/libraries/go/lib/pkg/icms-translate/translate/function/translate_container.go

@@ -446,6 +446,7 @@ func translateContainer(t CreationQueueMessage, tcfg TranslateConfig) (objs []me
 				return nil, err
 			}
 			pod.Spec.Containers = append(pod.Spec.Containers, llmCredentialManagerContainer)
+			applyLLMPodSecurityContext(pod, tcfg)
 		}
 
 		// Setup telemetry for the pod

src/libraries/go/lib/pkg/icms-translate/translate/function/translate_helm.go

@@ -366,6 +366,7 @@ func translateHelmChart(t CreationQueueMessage, tcfg TranslateConfig) (objs []me
 			return nil, err
 		}
 		utilsPod.Spec.Containers = append(utilsPod.Spec.Containers, llmCredentialManagerContainer)
+		applyLLMPodSecurityContext(utilsPod, tcfg)
 	}
 
 	if hasTelemetries {

src/libraries/go/lib/pkg/types/nvca/config/types.go

@@ -704,6 +704,10 @@ type WorkloadConfig struct {
 	DefaultStargateAddress string `yaml:",omitempty"`
 	StargateQUICInsecure   bool   `yaml:",omitempty"`
 
+	// LLMSidecarRunAsUser is the UID both LLM sidecars run as (default 1000).
+	// Negative leaves runAsUser unset so the platform (e.g. OpenShift) assigns.
+	LLMSidecarRunAsUser *int64 `yaml:",omitempty"`
+
 	// TransportTLS configures trust material used by workload transport clients.
 	TransportTLS *TransportTLSConfig `yaml:",omitempty"`
 }

src/libraries/go/lib/testdata/icms-translate/function/container/llm/exp.yaml

@@ -276,6 +276,9 @@ spec:
       requests:
         cpu: 500m
         memory: 512Mi
+    securityContext:
+      runAsGroup: 1000
+      runAsUser: 1000
     volumeMounts:
     - mountPath: /var/run/llm
       name: llm
@@ -342,6 +345,9 @@ spec:
       requests:
         cpu: 125m
         memory: 64Mi
+    securityContext:
+      runAsGroup: 1000
+      runAsUser: 1000
     volumeMounts:
     - mountPath: /var/run/llm
       name: llm
@@ -441,6 +447,8 @@ spec:
     - mountPath: /var/run/nvcf/info
       name: instance-data
   restartPolicy: Always
+  securityContext:
+    fsGroup: 1000
   serviceAccountName: my-sa
   terminationGracePeriodSeconds: 120
   tolerations: