Add UID-based nftables firewall for NATS and monit connections

agent/bootstrap.go

@@ -183,6 +183,10 @@ func (boot bootstrap) Run() (err error) { //nolint:gocyclo
 		}
 	}
 
+	if err = boot.platform.SetupFirewall(); err != nil {
+		return bosherr.WrapError(err, "Setting up firewall")
+	}
+
 	if err = boot.platform.SetupMonitUser(); err != nil {
 		return bosherr.WrapError(err, "Setting up monit user")
 	}

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/coreos/go-iptables v0.8.0
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/golang/mock v1.6.0
+	github.com/google/nftables v0.3.0
 	github.com/google/uuid v1.6.0
 	github.com/kevinburke/ssh_config v1.4.0
 	github.com/masterzen/winrm v0.0.0-20250927112105-5f8e6c707321
@@ -70,6 +71,8 @@ require (
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/klauspost/compress v1.18.3 // indirect
 	github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 // indirect
+	github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
+	github.com/mdlayher/socket v0.5.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.15 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect

go.sum

@@ -109,6 +109,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
+github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
 github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef h1:xpF9fUHpoIrrjX24DURVKiwHcFpw19ndIs+FwTSMbno=
 github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -157,6 +159,10 @@ github.com/masterzen/winrm v0.0.0-20250927112105-5f8e6c707321 h1:AKIJL2PfBX2uie0
 github.com/masterzen/winrm v0.0.0-20250927112105-5f8e6c707321/go.mod h1:JajVhkiG2bYSNYYPYuWG7WZHr42CTjMTcCjfInRNCqc=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.12.1 h1:D4O2wLxB384TS3ohBJMfolnxb4qGmoZ1PnWNtit8LYo=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.12.1/go.mod h1:RuJdxo0oI6dClIaMzdl3hewq3a065RH65dofJP03h8I=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
+github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI=
+github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI=
 github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
 github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -220,6 +226,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tidwall/transform v0.0.0-20201103190739-32f242e2dbde h1:AMNpJRc7P+GTwVbl8DkK2I9I8BBUzNiHuH/tlxrpan0=
 github.com/tidwall/transform v0.0.0-20201103190739-32f242e2dbde/go.mod h1:MvrEmduDUz4ST5pGZ7CABCnOU5f3ZiOAZzT6b1A6nX8=
+github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
+github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=

mbus/nats_handler.go

@@ -99,14 +99,29 @@ func NewNatsHandler(
 func (h *natsHandler) arpClean() {
 	connectionInfo, err := h.getConnectionInfo()
 	if err != nil {
-		h.logger.Error(h.logTag, "%v", bosherr.WrapError(err, "Getting connection info"))
+		h.logger.Error(h.logTag, "Failed to get connection info for ARP clean: %v", err)
+		return
 	}
-	err = h.platform.DeleteARPEntryWithIP(connectionInfo.IP)
-	if err != nil {
+	if err := h.platform.DeleteARPEntryWithIP(connectionInfo.IP); err != nil {
 		h.logger.Error(h.logTag, "Cleaning ip-mac address cache for: %s. Error: %v", connectionInfo.IP, err)
 	}
+}
 
-	h.logger.Debug(h.logTag, "Cleaned ip-mac address cache for: %s.", connectionInfo.IP)
+// updateFirewallForNATS calls the firewall hook to update NATS rules before connection/reconnection.
+// This allows DNS to be re-resolved, supporting HA failover where the director may have moved.
+func (h *natsHandler) updateFirewallForNATS() {
+	hook := h.platform.GetNatsFirewallHook()
+	if hook == nil {
+		return
+	}
+
+	settings := h.settingsService.GetSettings()
+	mbusURL := settings.GetMbusURL()
+
+	if err := hook.BeforeConnect(mbusURL); err != nil {
+		// Log but don't fail - firewall update failure shouldn't prevent connection attempt
+		h.logger.Warn(h.logTag, "Failed to update NATS firewall rules: %v", err)
+	}
 }
 
 func (h *natsHandler) Run(handlerFunc boshhandler.Func) error {
@@ -131,11 +146,21 @@ func (h *natsHandler) Start(handlerFunc boshhandler.Func) error {
 	if net.ParseIP(connectionInfo.IP) != nil {
 		h.arpClean()
 	}
+
+	// Update firewall rules before initial connection
+	h.updateFirewallForNATS()
+
 	var natsOptions = []nats.Option{
 		nats.RetryOnFailedConnect(true),
 		nats.DisconnectErrHandler(func(c *nats.Conn, err error) {
-			h.logger.Debug(natsHandlerLogTag, "Nats disconnected with Error: %v", err.Error())
+			if err != nil {
+				h.logger.Debug(natsHandlerLogTag, "Nats disconnected with Error: %v", err.Error())
+			} else {
+				h.logger.Debug(natsHandlerLogTag, "Nats disconnected")
+			}
 			h.logger.Debug(natsHandlerLogTag, "Attempting to reconnect: %v", c.IsReconnecting())
+			// Update firewall rules before reconnection attempts (allows DNS re-resolution)
+			h.updateFirewallForNATS()
 			for c.IsReconnecting() {
 				h.arpClean()
 				h.logger.Debug(natsHandlerLogTag, "Waiting to reconnect to nats.. Current attempt: %v, Connected: %v", c.Reconnects, c.IsConnected())
@@ -146,7 +171,11 @@ func (h *natsHandler) Start(handlerFunc boshhandler.Func) error {
 			h.logger.Debug(natsHandlerLogTag, "Reconnected to %v", c.ConnectedAddr())
 		}),
 		nats.ClosedHandler(func(c *nats.Conn) {
-			h.logger.Debug(natsHandlerLogTag, "Connection Closed with: %v", c.LastError().Error())
+			if err := c.LastError(); err != nil {
+				h.logger.Debug(natsHandlerLogTag, "Connection Closed with: %v", err.Error())
+			} else {
+				h.logger.Debug(natsHandlerLogTag, "Connection Closed")
+			}
 		}),
 		nats.ErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) {
 			h.logger.Debug(natsHandlerLogTag, err.Error())

mbus/nats_handler_test.go

@@ -18,6 +18,7 @@ import (
 	boshhandler "github.com/cloudfoundry/bosh-agent/v2/handler"
 	"github.com/cloudfoundry/bosh-agent/v2/mbus"
 	"github.com/cloudfoundry/bosh-agent/v2/mbus/mbusfakes"
+	"github.com/cloudfoundry/bosh-agent/v2/platform/firewall/firewallfakes"
 	"github.com/cloudfoundry/bosh-agent/v2/platform/platformfakes"
 	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
 	fakesettings "github.com/cloudfoundry/bosh-agent/v2/settings/fakes"
@@ -407,6 +408,54 @@ func init() { //nolint:funlen,gochecknoinits
 					})
 				})
 			})
+
+			Context("Firewall hook", func() {
+				var fakeFirewallHook *firewallfakes.FakeNatsFirewallHook
+
+				BeforeEach(func() {
+					fakeFirewallHook = &firewallfakes.FakeNatsFirewallHook{}
+					platform.GetNatsFirewallHookReturns(fakeFirewallHook)
+				})
+
+				It("calls GetNatsFirewallHook on Start", func() {
+					err := handler.Start(func(req boshhandler.Request) (res boshhandler.Response) { return })
+					Expect(err).NotTo(HaveOccurred())
+					defer handler.Stop()
+
+					Expect(platform.GetNatsFirewallHookCallCount()).To(BeNumerically(">=", 1))
+				})
+
+				It("calls BeforeConnect with the mbus URL before initial connection", func() {
+					err := handler.Start(func(req boshhandler.Request) (res boshhandler.Response) { return })
+					Expect(err).NotTo(HaveOccurred())
+					defer handler.Stop()
+
+					Expect(fakeFirewallHook.BeforeConnectCallCount()).To(Equal(1))
+					mbusURL := fakeFirewallHook.BeforeConnectArgsForCall(0)
+					Expect(mbusURL).To(Equal("nats://fake-username:fake-password@127.0.0.1:1234"))
+				})
+
+				It("does not fail if hook returns nil", func() {
+					platform.GetNatsFirewallHookReturns(nil)
+
+					err := handler.Start(func(req boshhandler.Request) (res boshhandler.Response) { return })
+					Expect(err).NotTo(HaveOccurred())
+					defer handler.Stop()
+				})
+
+				It("logs warning but does not fail if BeforeConnect returns error", func() {
+					fakeFirewallHook.BeforeConnectReturns(errors.New("firewall update failed"))
+					loggerOutBuf = bytes.NewBufferString("")
+					logger = boshlog.NewWriterLogger(boshlog.LevelWarn, loggerOutBuf)
+					handler = mbus.NewNatsHandler(settingsService, connector, logger, platform)
+
+					err := handler.Start(func(req boshhandler.Request) (res boshhandler.Response) { return })
+					Expect(err).NotTo(HaveOccurred())
+					defer handler.Stop()
+
+					Expect(loggerOutBuf.String()).To(ContainSubstring("Failed to update NATS firewall rules"))
+				})
+			})
 		})
 
 		Describe("Send", func() {

platform/dummy_platform.go

@@ -15,6 +15,7 @@ import (
 	boshlogstarprovider "github.com/cloudfoundry/bosh-agent/v2/agent/logstarprovider"
 	boshdpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver"
 	boshcert "github.com/cloudfoundry/bosh-agent/v2/platform/cert"
+	"github.com/cloudfoundry/bosh-agent/v2/platform/firewall"
 	boship "github.com/cloudfoundry/bosh-agent/v2/platform/net/ip"
 	boshstats "github.com/cloudfoundry/bosh-agent/v2/platform/stats"
 	boshvitals "github.com/cloudfoundry/bosh-agent/v2/platform/vitals"
@@ -562,6 +563,14 @@ func (p dummyPlatform) SetupRecordsJSONPermission(path string) error {
 	return nil
 }
 
+func (p dummyPlatform) SetupFirewall() error {
+	return nil
+}
+
 func (p dummyPlatform) Shutdown() error {
 	return nil
 }
+
+func (p dummyPlatform) GetNatsFirewallHook() firewall.NatsFirewallHook {
+	return nil
+}

platform/firewall/firewall.go

@@ -0,0 +1,39 @@
+// Package firewall provides nftables-based firewall management for the BOSH agent.
+//
+// The firewall protects access to:
+// - Monit (port 2822 on localhost): Used by the agent to manage job processes
+// - NATS (director's message bus): Used for agent-director communication
+//
+// Security Model:
+// The firewall uses UID-based matching (meta skuid 0) to allow only root processes
+// to access these services. This blocks non-root BOSH job workloads (vcap user)
+// while allowing the agent and operators to access monit/NATS.
+//
+// This approach is simpler and more reliable than cgroup-based matching, which
+// fails in nested container environments due to cgroup filesystem bind-mount issues.
+package firewall
+
+// Manager handles firewall setup
+//
+//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
+//counterfeiter:generate . Manager
+type Manager interface {
+	// SetupMonitFirewall creates firewall rules to protect monit (port 2822).
+	// Only root (UID 0) is allowed to connect.
+	SetupMonitFirewall() error
+
+	// SetupNATSFirewall creates firewall rules to protect NATS.
+	// Only root (UID 0) is allowed to connect to the resolved NATS address.
+	// This method resolves DNS and should be called before each connection attempt.
+	SetupNATSFirewall(mbusURL string) error
+}
+
+// NatsFirewallHook is called by the NATS handler before connection/reconnection.
+// This allows DNS to be re-resolved, supporting HA failover scenarios.
+//
+//counterfeiter:generate . NatsFirewallHook
+type NatsFirewallHook interface {
+	// BeforeConnect is called before each NATS connection/reconnection attempt.
+	// It resolves the NATS URL and updates firewall rules with the resolved IP.
+	BeforeConnect(mbusURL string) error
+}

platform/firewall/firewall_suite_test.go

@@ -0,0 +1,15 @@
+//go:build linux
+
+package firewall_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestFirewall(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Firewall Suite")
+}