fix: initialize node uuid during enrollment

deployments/helm/fleet-intelligence-agent/templates/daemonset.yaml

@@ -101,6 +101,9 @@ spec:
           volumeMounts:
             - name: dev
               mountPath: /dev
+            - name: sys
+              mountPath: /sys
+              readOnly: true
             - name: state
               mountPath: /var/lib/fleetint
       {{- else if .Values.enroll.unenroll }}

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/prometheus/client_golang v1.23.0
+	github.com/prometheus/procfs v0.16.1
 	github.com/stretchr/testify v1.11.1
 	github.com/urfave/cli v1.22.17
 	go.opentelemetry.io/proto/otlp v1.8.0
@@ -76,7 +77,6 @@ require (
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect
 	github.com/quic-go/quic-go v0.57.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect

internal/agentstate/sqlite.go

@@ -112,6 +112,85 @@ func (s *sqliteState) SetNodeUUID(ctx context.Context, value string) error {
 	return s.setMetadata(ctx, pkgmetadata.MetadataKeyMachineID, value)
 }
 
+func (s *sqliteState) GetOrCreateNodeUUID(ctx context.Context, create func() (string, error)) (string, bool, error) {
+	db, err := s.openReadWrite()
+	if err != nil {
+		return "", false, err
+	}
+	defer db.Close()
+
+	if stateFile, err := s.stateFileFn(); err == nil {
+		if err := config.SecureStateFilePermissions(stateFile); err != nil {
+			return "", false, fmt.Errorf("secure state file permissions: %w", err)
+		}
+	}
+	if err := pkgmetadata.CreateTableMetadata(ctx, db); err != nil {
+		return "", false, fmt.Errorf("create metadata table: %w", err)
+	}
+
+	return GetOrCreateNodeUUIDMetadata(ctx, db, create)
+}
+
+// GetOrCreateNodeUUIDMetadata atomically returns the committed node UUID
+// metadata value, creating it when missing or repairing an empty value.
+func GetOrCreateNodeUUIDMetadata(ctx context.Context, db *sql.DB, create func() (string, error)) (string, bool, error) {
+	if create == nil {
+		return "", false, fmt.Errorf("node UUID create function is required")
+	}
+
+	existing, ok, err := readNodeUUIDDB(ctx, db)
+	if err != nil {
+		return "", false, err
+	} else if ok {
+		return existing, false, nil
+	}
+
+	candidate, err := create()
+	if err != nil {
+		return "", false, err
+	}
+	if candidate == "" {
+		return "", false, fmt.Errorf("created node UUID is empty")
+	}
+	if _, err := db.ExecContext(ctx, `
+INSERT INTO gpud_metadata (key, value) VALUES (?, ?)
+ON CONFLICT(key) DO UPDATE SET value = excluded.value
+WHERE gpud_metadata.value = ''`,
+		pkgmetadata.MetadataKeyMachineID,
+		candidate,
+	); err != nil {
+		return "", false, fmt.Errorf("insert node UUID metadata: %w", err)
+	}
+
+	committed, ok, err := readNodeUUIDDB(ctx, db)
+	if err != nil {
+		return "", false, err
+	} else if !ok {
+		return "", false, fmt.Errorf("node UUID metadata was not committed")
+	} else if committed == "" {
+		return "", false, fmt.Errorf("node UUID metadata is empty")
+	}
+	return committed, committed == candidate, nil
+}
+
+func readNodeUUIDDB(ctx context.Context, db *sql.DB) (string, bool, error) {
+	var value string
+	err := db.QueryRowContext(ctx, `
+SELECT value FROM gpud_metadata WHERE key = ?`,
+		pkgmetadata.MetadataKeyMachineID,
+	).Scan(&value)
+	switch {
+	case err == nil && value != "":
+		return value, true, nil
+	case err == nil:
+		return "", false, nil
+	case errors.Is(err, sql.ErrNoRows):
+		return "", false, nil
+	default:
+		return "", false, fmt.Errorf("read node UUID metadata: %w", err)
+	}
+}
+
 func (s *sqliteState) GetNodeGroup(ctx context.Context) (string, bool, error) {
 	return s.getMetadata(ctx, MetadataKeyNodeGroup)
 }
@@ -174,8 +253,7 @@ func (s *sqliteState) setMetadata(ctx context.Context, key, value string) error
 	}
 	defer db.Close()
 
-	stateFile, err := s.stateFileFn()
-	if err == nil {
+	if stateFile, err := s.stateFileFn(); err == nil {
 		if err := config.SecureStateFilePermissions(stateFile); err != nil {
 			return fmt.Errorf("secure state file permissions: %w", err)
 		}

internal/agentstate/sqlite_test.go

@@ -120,6 +120,44 @@ func TestSQLiteStateMissingValue(t *testing.T) {
 	require.Empty(t, value)
 }
 
+func TestSQLiteStateGetOrCreateNodeUUID(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	state := newTestSQLiteState(t)
+
+	value, created, err := state.GetOrCreateNodeUUID(ctx, func() (string, error) {
+		return "node-created", nil
+	})
+	require.NoError(t, err)
+	require.True(t, created)
+	require.Equal(t, "node-created", value)
+
+	value, created, err = state.GetOrCreateNodeUUID(ctx, func() (string, error) {
+		t.Fatal("create should not run when node UUID is already persisted")
+		return "", nil
+	})
+	require.NoError(t, err)
+	require.False(t, created)
+	require.Equal(t, "node-created", value)
+
+	emptyRowState := newTestSQLiteState(t)
+	err = emptyRowState.SetNodeUUID(ctx, "")
+	require.NoError(t, err)
+
+	value, created, err = emptyRowState.GetOrCreateNodeUUID(ctx, func() (string, error) {
+		return "node-repaired", nil
+	})
+	require.NoError(t, err)
+	require.True(t, created)
+	require.Equal(t, "node-repaired", value)
+
+	value, ok, err := emptyRowState.GetNodeUUID(ctx)
+	require.NoError(t, err)
+	require.True(t, ok)
+	require.Equal(t, "node-repaired", value)
+}
+
 func TestSQLiteStateMissingMetadataTableIsTreatedAsAbsent(t *testing.T) {
 	t.Parallel()
 

internal/agentstate/state.go

@@ -40,8 +40,11 @@ type State interface {
 	GetSAK(ctx context.Context) (value string, ok bool, err error)
 	SetSAK(ctx context.Context, value string) error
 
+	// GetNodeUUID reads the committed node UUID without creating one.
 	GetNodeUUID(ctx context.Context) (value string, ok bool, err error)
 	SetNodeUUID(ctx context.Context, value string) error
+	// GetOrCreateNodeUUID creates and commits the node UUID when no value exists.
+	GetOrCreateNodeUUID(ctx context.Context, create func() (string, error)) (value string, created bool, err error)
 
 	GetEnrollmentTime(ctx context.Context) (value time.Time, ok bool, err error)
 	SetEnrollmentTime(ctx context.Context, value time.Time) error

internal/attestation/backend_test.go

@@ -51,6 +51,21 @@ func (s *stubState) GetNodeUUID(context.Context) (string, bool, error) {
 	return s.nodeUUID, s.nodeOK, s.nodeErr
 }
 func (s *stubState) SetNodeUUID(context.Context, string) error { return nil }
+func (s *stubState) GetOrCreateNodeUUID(_ context.Context, create func() (string, error)) (string, bool, error) {
+	if s.nodeErr != nil {
+		return "", false, s.nodeErr
+	}
+	if s.nodeOK && s.nodeUUID != "" {
+		return s.nodeUUID, false, nil
+	}
+	value, err := create()
+	if err != nil {
+		return "", false, err
+	}
+	s.nodeUUID = value
+	s.nodeOK = true
+	return value, true, nil
+}
 func (s *stubState) GetNodeGroup(context.Context) (string, bool, error) {
 	return "", false, nil
 }

internal/enrollment/enrollment.go

@@ -36,12 +36,15 @@ import (
 	inventorysink "github.com/NVIDIA/fleet-intelligence-agent/internal/inventory/sink"
 	inventorysource "github.com/NVIDIA/fleet-intelligence-agent/internal/inventory/source"
 	"github.com/NVIDIA/fleet-intelligence-agent/internal/machineinfo"
+	"github.com/NVIDIA/fleet-intelligence-agent/internal/nodeidentity"
 	"github.com/NVIDIA/fleet-intelligence-agent/internal/registry"
 )
 
 var (
 	newBackendClient               = backendclient.New
 	syncInventoryAfterEnroll       = syncInventoryOnce
+	ensureNodeUUID                 = nodeidentity.EnsureNodeUUID
+	storeEnrollmentConfig          = storeConfigInMetadata
 	postEnrollInventorySyncTimeout = time.Minute
 )
 
@@ -76,8 +79,11 @@ func EnrollWithConfigAndMetadata(ctx context.Context, baseEndpoint, sakToken str
 	if err != nil {
 		return err
 	}
+	if _, err := ensureNodeUUID(ctx, agentstate.NewSQLite()); err != nil {
+		return fmt.Errorf("failed to initialize node UUID: %w", err)
+	}
 	enrolledAt := time.Now().UTC()
-	if err := storeConfigInMetadata(ctx, baseURL.String(), jwtToken, sakToken, enrolledAt, normalizedEnrollMetadata(metadata)); err != nil {
+	if err := storeEnrollmentConfig(ctx, baseURL.String(), jwtToken, sakToken, enrolledAt, normalizedEnrollMetadata(metadata)); err != nil {
 		return fmt.Errorf("failed to store configuration: %w", err)
 	}
 	syncCtx, cancel := context.WithTimeout(ctx, postEnrollInventorySyncTimeout)

internal/enrollment/enrollment_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/NVIDIA/fleet-intelligence-agent/internal/agentstate"
 	"github.com/NVIDIA/fleet-intelligence-agent/internal/backendclient"
 	"github.com/NVIDIA/fleet-intelligence-agent/internal/config"
+	"github.com/NVIDIA/fleet-intelligence-agent/internal/nodeidentity"
 )
 
 type fakeBackendClient struct {
@@ -60,9 +61,11 @@ func (f *fakeBackendClient) SubmitAttestation(context.Context, string, *backendc
 func TestEnrollWorkflow(t *testing.T) {
 	originalFactory := newBackendClient
 	originalSync := syncInventoryAfterEnroll
+	originalEnsure := ensureNodeUUID
 	t.Cleanup(func() {
 		newBackendClient = originalFactory
 		syncInventoryAfterEnroll = originalSync
+		ensureNodeUUID = originalEnsure
 	})
 
 	client := &fakeBackendClient{enrollJWT: "jwt-token"}
@@ -72,7 +75,14 @@ func TestEnrollWorkflow(t *testing.T) {
 	}
 
 	syncCalled := false
+	ensureCalled := false
+	ensureNodeUUID = func(ctx context.Context, state nodeidentity.State) (string, error) {
+		require.NotNil(t, state)
+		ensureCalled = true
+		return "4c4c4544-0053-5210-8038-c8c04f583034", nil
+	}
 	syncInventoryAfterEnroll = func(ctx context.Context, cfg *config.Config) error {
+		require.True(t, ensureCalled, "node UUID should be initialized before post-enroll inventory sync")
 		syncCalled = true
 		return nil
 	}
@@ -83,6 +93,7 @@ func TestEnrollWorkflow(t *testing.T) {
 	err := Enroll(context.Background(), "https://example.com", "sak-token")
 	require.NoError(t, err)
 	require.Equal(t, "sak-token", client.enrollSAK)
+	require.True(t, ensureCalled)
 	require.True(t, syncCalled)
 }
 
@@ -340,6 +351,39 @@ func TestEnrollWorkflowErrors(t *testing.T) {
 		require.ErrorContains(t, err, "enroll boom")
 	})
 
+	t.Run("node UUID initialization", func(t *testing.T) {
+		originalFactory := newBackendClient
+		originalEnsure := ensureNodeUUID
+		originalSync := syncInventoryAfterEnroll
+		originalStore := storeEnrollmentConfig
+		t.Cleanup(func() {
+			newBackendClient = originalFactory
+			ensureNodeUUID = originalEnsure
+			syncInventoryAfterEnroll = originalSync
+			storeEnrollmentConfig = originalStore
+		})
+		newBackendClient = func(rawBaseURL string) (backendclient.Client, error) {
+			return &fakeBackendClient{enrollJWT: "jwt-token"}, nil
+		}
+		ensureNodeUUID = func(context.Context, nodeidentity.State) (string, error) {
+			return "", errors.New("identity store failed")
+		}
+		syncInventoryAfterEnroll = func(context.Context, *config.Config) error {
+			t.Fatal("inventory sync should not run when node UUID initialization fails")
+			return nil
+		}
+		storeEnrollmentConfig = func(context.Context, string, string, string, time.Time, EnrollMetadata) error {
+			t.Fatal("enrollment metadata should not be stored when node UUID initialization fails")
+			return nil
+		}
+
+		tmpHome := t.TempDir()
+		t.Setenv("HOME", tmpHome)
+
+		err := Enroll(context.Background(), "https://example.com", "sak-token")
+		require.ErrorContains(t, err, "failed to initialize node UUID")
+	})
+
 	t.Run("localhost legacy endpoint allowed", func(t *testing.T) {
 		originalFactory := newBackendClient
 		originalSync := syncInventoryAfterEnroll

internal/inventory/sink/backend_test.go

@@ -63,6 +63,20 @@ func (f *fakeState) GetNodeUUID(context.Context) (string, bool, error) {
 	return f.nodeUUID, f.nodeUUID != "", nil
 }
 func (f *fakeState) SetNodeUUID(context.Context, string) error { return nil }
+func (f *fakeState) GetOrCreateNodeUUID(_ context.Context, create func() (string, error)) (string, bool, error) {
+	if f.err != nil {
+		return "", false, f.err
+	}
+	if f.nodeUUID != "" {
+		return f.nodeUUID, false, nil
+	}
+	value, err := create()
+	if err != nil {
+		return "", false, err
+	}
+	f.nodeUUID = value
+	return value, true, nil
+}
 func (f *fakeState) GetNodeGroup(context.Context) (string, bool, error) {
 	if f.nodeGroupErr != nil {
 		return "", false, f.nodeGroupErr