From 4031547a2d3d41b91aee30b747c11b9ddbbcb8ca Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 30 Jun 2026 16:25:11 +0800 Subject: [PATCH] feat: add --drop-internal-access for VM-to-VM isolation VM-to-VM isolation no longer asks operators to restate the cocoon subnet as a --drop-cidr: --drop-internal-access derives the FORWARD DROP target from the subnet cocoon already knows. --drop-cidr now covers only external ranges (e.g. management networks). Same-node VMs share cni0 and are L2-switched, which bypasses iptables unless bridge-nf-call-iptables=1. Node setup now loads br_netfilter and enables it (verifying, failing closed) whenever a drop target is set, so the isolation is never silently a no-op. Drop CIDRs are validated and canonicalized up front; IPv6 is rejected. The DROP rules are tagged cocoon-net-drop so teardown removes exactly them. --- README.md | 31 ++++++++ cmd/adopt.go | 34 +++++---- cmd/daemon.go | 12 +-- cmd/helpers.go | 22 +++--- cmd/init.go | 34 +++++---- cmd/teardown.go | 5 ++ node/iptables_linux.go | 165 +++++++++++++++++++++++++++++++++++++++-- node/iptables_stub.go | 5 +- node/node.go | 12 ++- node/sysctl.go | 8 +- pool/pool.go | 6 ++ 11 files changed, 283 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index fb49733..0e80878 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,37 @@ sudo cocoon-net init \ --pool-size 140 ``` +#### VM egress isolation (`--drop-internal-access`, `--drop-cidr`) + +Both flags (accepted by `init` and `adopt`) block VM-originated traffic and are +persisted to `pool.json`, reapplied by the daemon as `FORWARD` DROP rules at the +head of the chain so they win over the default accept rules. Return traffic and +internet egress are unaffected. + +- `--drop-internal-access` blocks **VM-to-VM** traffic within the cocoon subnet. + cocoon-net already knows the subnet from `--subnet`, so there is no CIDR to + restate. +- `--drop-cidr` (repeatable) blocks additional **external** destination ranges, + e.g. internal/VPC management networks. + +Same-node VMs share `cni0` and are switched at L2, which bypasses iptables +unless `bridge-nf-call-iptables=1`. When either flag is set, node setup loads +`br_netfilter` and enables that toggle, **failing closed** if it cannot — so the +isolation is never silently a no-op. The DROP rules are tagged `cocoon-net-drop`, +so `teardown` removes exactly them. + +```bash +sudo cocoon-net init \ + --platform gke --node-name cocoon-pool \ + --subnet 172.20.100.0/24 --pool-size 140 \ + --drop-internal-access \ + --drop-cidr 10.0.0.0/8 +``` + +> Note: traffic to the node's own address (e.g. a kubelet bound on the cni0 +> gateway IP) is delivered via `INPUT`, not `FORWARD`, so these flags do not +> cover it — restrict those separately (host `INPUT` rule or bind off cni0). + ### daemon -- run DHCP server (systemd service) ```bash diff --git a/cmd/adopt.go b/cmd/adopt.go index f5765ee..b7f3417 100644 --- a/cmd/adopt.go +++ b/cmd/adopt.go @@ -73,6 +73,8 @@ func runAdopt(cmd *cobra.Command, _ []string) error { fmt.Printf(" pool-size: 0\n") } fmt.Printf(" dns: %s\n", strings.Join(dnsServers, ",")) + fmt.Printf(" drop-internal: %v\n", flagDropInternal) + fmt.Printf(" drop-cidr: %s\n", strings.Join(flagDropCIDRs, ",")) fmt.Printf(" state-dir: %s\n", flagStateDir) fmt.Printf(" manage-iptables: %v\n", flagManageIPTables) fmt.Println() @@ -90,11 +92,13 @@ func runAdopt(cmd *cobra.Command, _ []string) error { } nodeCfg := &node.Config{ - Gateway: gateway, - SubnetCIDR: flagSubnet, - PrimaryNIC: primaryNIC, - SecondaryNICs: secondaryNICs, - SkipIPTables: !flagManageIPTables, + Gateway: gateway, + SubnetCIDR: flagSubnet, + PrimaryNIC: primaryNIC, + SecondaryNICs: secondaryNICs, + SkipIPTables: !flagManageIPTables, + DropInternalAccess: flagDropInternal, + DropCIDRs: flagDropCIDRs, } if err := node.Setup(ctx, nodeCfg); err != nil { return fmt.Errorf("node setup: %w", err) @@ -102,15 +106,17 @@ func runAdopt(cmd *cobra.Command, _ []string) error { logger.Info(ctx, "node networking configured (adopted, cloud side untouched)") state := &pool.State{ - Platform: platformName, - NodeName: flagNodeName, - Subnet: flagSubnet, - Gateway: gateway, - PrimaryNIC: primaryNIC, - SecondaryNICs: secondaryNICs, - IPs: ips, - DNSServers: dnsServers, - StateDir: flagStateDir, + Platform: platformName, + NodeName: flagNodeName, + Subnet: flagSubnet, + Gateway: gateway, + PrimaryNIC: primaryNIC, + SecondaryNICs: secondaryNICs, + IPs: ips, + DNSServers: dnsServers, + DropInternalAccess: flagDropInternal, + DropCIDRs: flagDropCIDRs, + StateDir: flagStateDir, } if err := state.Save(ctx); err != nil { return fmt.Errorf("save pool state: %w", err) diff --git a/cmd/daemon.go b/cmd/daemon.go index 667a920..08015ae 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -65,11 +65,13 @@ func runDaemon(cmd *cobra.Command, _ []string) error { primaryNIC = platform.DefaultNIC(state.Platform) } if setupErr := node.Setup(ctx, &node.Config{ - Gateway: state.Gateway, - SubnetCIDR: state.Subnet, - PrimaryNIC: primaryNIC, - SecondaryNICs: state.SecondaryNICs, - SkipIPTables: flagSkipIPTables, + Gateway: state.Gateway, + SubnetCIDR: state.Subnet, + PrimaryNIC: primaryNIC, + SecondaryNICs: state.SecondaryNICs, + SkipIPTables: flagSkipIPTables, + DropInternalAccess: state.DropInternalAccess, + DropCIDRs: state.DropCIDRs, }); setupErr != nil { return fmt.Errorf("node setup: %w", setupErr) } diff --git a/cmd/helpers.go b/cmd/helpers.go index 87fe75e..7215a74 100644 --- a/cmd/helpers.go +++ b/cmd/helpers.go @@ -15,15 +15,17 @@ import ( const defaultStateDir = "/var/lib/cocoon/net" var ( - flagPlatform string - flagNodeName string - flagSubnet string - flagPoolSize int - flagGateway string - flagPrimaryNIC string - flagDNS string - flagStateDir string - flagDryRun bool + flagPlatform string + flagNodeName string + flagSubnet string + flagPoolSize int + flagGateway string + flagPrimaryNIC string + flagDNS string + flagStateDir string + flagDryRun bool + flagDropInternal bool + flagDropCIDRs []string // flagManageIPTables is the inverse of node.Config.SkipIPTables, // exposed only on adopt (off by default to preserve host rules). @@ -41,6 +43,8 @@ func registerCommonFlags(cmd *cobra.Command, defaultPoolSize int) { cmd.Flags().StringVar(&flagDNS, "dns", "8.8.8.8,1.1.1.1", "comma-separated DNS servers for DHCP clients") cmd.Flags().StringVar(&flagStateDir, "state-dir", defaultStateDir, "state directory") cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "show what would be done without making changes") + cmd.Flags().BoolVar(&flagDropInternal, "drop-internal-access", false, "block VM-to-VM traffic within the cocoon subnet") + cmd.Flags().StringArrayVar(&flagDropCIDRs, "drop-cidr", nil, "additional external destination CIDR VMs may not reach; repeatable (e.g. --drop-cidr 10.0.0.0/8)") _ = cmd.MarkFlagRequired("node-name") _ = cmd.MarkFlagRequired("subnet") } diff --git a/cmd/init.go b/cmd/init.go index ef47cbd..fe501ab 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -49,6 +49,8 @@ func runInit(cmd *cobra.Command, _ []string) error { fmt.Printf(" subnet: %s\n", cfg.SubnetCIDR) fmt.Printf(" pool-size: %d\n", cfg.PoolSize) fmt.Printf(" dns: %s\n", strings.Join(cfg.DNSServers, ",")) + fmt.Printf(" drop-internal: %v\n", flagDropInternal) + fmt.Printf(" drop-cidr: %s\n", strings.Join(flagDropCIDRs, ",")) fmt.Printf(" state-dir: %s\n", flagStateDir) return nil } @@ -66,10 +68,12 @@ func runInit(cmd *cobra.Command, _ []string) error { logger.Infof(ctx, "provisioned %d IPs on subnet %s", len(result.IPs), result.SubnetCIDR) nodeCfg := &node.Config{ - Gateway: result.Gateway, - SubnetCIDR: result.SubnetCIDR, - PrimaryNIC: result.PrimaryNIC, - SecondaryNICs: result.SecondaryNICs, + Gateway: result.Gateway, + SubnetCIDR: result.SubnetCIDR, + PrimaryNIC: result.PrimaryNIC, + SecondaryNICs: result.SecondaryNICs, + DropInternalAccess: flagDropInternal, + DropCIDRs: flagDropCIDRs, } if err := node.Setup(ctx, nodeCfg); err != nil { return fmt.Errorf("node setup: %w", err) @@ -77,16 +81,18 @@ func runInit(cmd *cobra.Command, _ []string) error { logger.Info(ctx, "node networking configured") state := &pool.State{ - Platform: result.Platform, - NodeName: cfg.NodeName, - Subnet: result.SubnetCIDR, - Gateway: result.Gateway, - PrimaryNIC: result.PrimaryNIC, - SecondaryNICs: result.SecondaryNICs, - IPs: result.IPs, - AliasRangeName: result.AliasRangeName, - DNSServers: dnsServers, - StateDir: flagStateDir, + Platform: result.Platform, + NodeName: cfg.NodeName, + Subnet: result.SubnetCIDR, + Gateway: result.Gateway, + PrimaryNIC: result.PrimaryNIC, + SecondaryNICs: result.SecondaryNICs, + IPs: result.IPs, + AliasRangeName: result.AliasRangeName, + DNSServers: dnsServers, + DropInternalAccess: flagDropInternal, + DropCIDRs: flagDropCIDRs, + StateDir: flagStateDir, } if err := state.Save(ctx); err != nil { return fmt.Errorf("save pool state: %w", err) diff --git a/cmd/teardown.go b/cmd/teardown.go index 77c7d8f..af98bc4 100644 --- a/cmd/teardown.go +++ b/cmd/teardown.go @@ -10,6 +10,7 @@ import ( "github.com/projecteru2/core/log" "github.com/spf13/cobra" + "github.com/cocoonstack/cocoon-net/node" "github.com/cocoonstack/cocoon-net/platform" ) @@ -57,6 +58,10 @@ func runTeardown(cmd *cobra.Command, _ []string) error { } logger.Infof(ctx, "teardown complete for %s", state.Platform) + if err := node.ClearDropRules(ctx); err != nil { + logger.Warnf(ctx, "clear iptables drop rules: %v", err) + } + if err := state.Delete(ctx); err != nil { logger.Warnf(ctx, "delete pool state: %v", err) } diff --git a/node/iptables_linux.go b/node/iptables_linux.go index 2689a53..3759585 100644 --- a/node/iptables_linux.go +++ b/node/iptables_linux.go @@ -5,16 +5,69 @@ package node import ( "context" "fmt" + "net" + "os" + "os/exec" + "strings" "github.com/coreos/go-iptables/iptables" "github.com/projecteru2/core/log" ) -// setupIPTables installs FORWARD rules between secondary NICs and the bridge, -// and a NAT MASQUERADE rule for outbound VM traffic. -func setupIPTables(ctx context.Context, subnetCIDR string, secondaryNICs []string) error { +// dropRuleComment tags cocoon-net's DROP rules for teardown. Must stay +// quote-safe ([-_+./0-9A-Za-z]) or iptables -S quotes it, breaking removal. +const dropRuleComment = "cocoon-net-drop" + +// ClearDropRules removes the FORWARD egress-isolation rules cocoon-net installed. +func ClearDropRules(ctx context.Context) error { + logger := log.WithFunc("node.ClearDropRules") + + ipt, err := iptables.New() + if err != nil { + return fmt.Errorf("init iptables: %w", err) + } + rules, err := ipt.List("filter", "FORWARD") + if err != nil { + return fmt.Errorf("list FORWARD: %w", err) + } + + removed, failed := 0, 0 + for _, rule := range rules { + if !strings.Contains(rule, dropRuleComment) { + continue + } + // List emits "-A FORWARD "; Delete wants only . + fields := strings.Fields(rule) + if len(fields) < 3 { + continue + } + if err := ipt.Delete("filter", "FORWARD", fields[2:]...); err != nil { + failed++ + continue + } + removed++ + } + + logger.Infof(ctx, "cleared %d egress drop rule(s)", removed) + if failed > 0 { + return fmt.Errorf("delete %d of %d drop rules failed", failed, removed+failed) + } + return nil +} + +// setupIPTables installs the FORWARD rules between secondary NICs and the +// bridge, a NAT MASQUERADE rule for outbound VM traffic, and egress DROP rules +// isolating VMs from their own subnet (dropInternal) and from dropCIDRs. +func setupIPTables(ctx context.Context, subnetCIDR string, secondaryNICs []string, dropInternal bool, dropCIDRs []string) error { logger := log.WithFunc("node.setupIPTables") + // Resolve and validate the drop targets before installing any rule, so a + // bad CIDR fails without leaving the chain half-configured. + dropTargets, err := resolveDropTargets(subnetCIDR, dropInternal, dropCIDRs) + if err != nil { + return err + } + ipt, err := iptables.New() if err != nil { return fmt.Errorf("init iptables: %w", err) @@ -37,11 +90,97 @@ func setupIPTables(ctx context.Context, subnetCIDR string, secondaryNICs []strin return fmt.Errorf("iptables NAT MASQUERADE: %w", err) } - logger.Infof(ctx, "iptables configured for subnet %s", subnetCIDR) + if len(dropTargets) == 0 { + logger.Infof(ctx, "iptables configured for subnet %s", subnetCIDR) + return nil + } + + // Same-node VMs share cni0 and are switched at L2, which bypasses iptables + // unless bridge-nf-call-iptables is on. Without it the DROP rules below + // would silently not apply to VM-to-VM, so enable it (fail closed) first. + if err := ensureBridgeNFCall(ctx); err != nil { + return fmt.Errorf("enable bridge netfilter: %w", err) + } + + // Insert at the head of FORWARD so DROP wins over the ACCEPT rules above. + // The -i BridgeName match leaves return traffic (no -i cni0) alone; VM + // traffic to the gateway is host-bound via INPUT, not FORWARD, so unaffected. + for _, dst := range dropTargets { + if err := iptInsert(ipt, "filter", "FORWARD", "-i", BridgeName, "-d", dst, "-m", "comment", "--comment", dropRuleComment, "-j", "DROP"); err != nil { + return fmt.Errorf("iptables FORWARD drop %s: %w", dst, err) + } + } + + logger.Infof(ctx, "iptables configured for subnet %s, %d egress drop rule(s)", subnetCIDR, len(dropTargets)) + return nil +} + +// resolveDropTargets resolves the CIDRs VM egress is blocked from reaching: the +// subnet itself when dropInternal is set (VM-to-VM isolation, reusing the range +// cocoon already knows), plus operator-supplied dropCIDRs. CIDRs are +// canonicalized so iptInsert's existence check dedups; IPv6 is rejected because +// the rules go through the IPv4 iptables binary. +func resolveDropTargets(subnetCIDR string, dropInternal bool, dropCIDRs []string) ([]string, error) { + var raw []string + if dropInternal { + raw = append(raw, subnetCIDR) + } + raw = append(raw, dropCIDRs...) + + out := make([]string, 0, len(raw)) + for _, cidr := range raw { + ip, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("invalid drop CIDR %q: %w", cidr, err) + } + if ip.To4() == nil { + return nil, fmt.Errorf("drop CIDR %q must be IPv4", cidr) + } + out = append(out, ipNet.String()) + } + return out, nil +} + +// ensureBridgeNFCall loads br_netfilter and turns on bridge-nf-call-iptables so +// same-bridge (same-node VM-to-VM) frames traverse iptables. It verifies the +// toggle stuck, failing closed rather than leaving DROP rules the kernel would +// quietly skip for L2-switched traffic. +func ensureBridgeNFCall(ctx context.Context) error { + logger := log.WithFunc("node.ensureBridgeNFCall") + + // modprobe is the authoritative loader for a kernel module and its deps, + // with no stdlib equivalent; it is a no-op when br_netfilter is loaded. + logger.Debug(ctx, "running modprobe br_netfilter (external binary)") + if out, err := exec.CommandContext(ctx, "modprobe", "br_netfilter").CombinedOutput(); err != nil { + return fmt.Errorf("modprobe br_netfilter (%s): %w", strings.TrimSpace(string(out)), err) + } + + const key = "net.bridge.bridge-nf-call-iptables" + if err := writeSysctl(key, "1"); err != nil { + return fmt.Errorf("write sysctl %s=1: %w", key, err) + } + got, err := readSysctl(key) + if err != nil { + return fmt.Errorf("read sysctl %s: %w", key, err) + } + if got != "1" { + return fmt.Errorf("sysctl %s is %q after write, want 1", key, got) + } + + logger.Info(ctx, "br_netfilter loaded, bridge-nf-call-iptables=1") return nil } -// iptEnsure adds an iptables rule if it does not already exist. +// readSysctl reads a sysctl value via /proc/sys, trimming surrounding whitespace. +func readSysctl(key string) (string, error) { + b, err := os.ReadFile(sysctlPath(key)) //nolint:gosec // sysctl read of a known proc path + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +// iptEnsure appends an iptables rule if it does not already exist. func iptEnsure(ipt *iptables.IPTables, table, chain string, args ...string) error { exists, err := ipt.Exists(table, chain, args...) if err != nil { @@ -55,3 +194,19 @@ func iptEnsure(ipt *iptables.IPTables, table, chain string, args ...string) erro } return nil } + +// iptInsert inserts an iptables rule at the head of the chain if it does not +// already exist, so the rule takes precedence over appended rules. +func iptInsert(ipt *iptables.IPTables, table, chain string, args ...string) error { + exists, err := ipt.Exists(table, chain, args...) + if err != nil { + return fmt.Errorf("check rule: %w", err) + } + if exists { + return nil + } + if err := ipt.Insert(table, chain, 1, args...); err != nil { + return fmt.Errorf("insert rule: %w", err) + } + return nil +} diff --git a/node/iptables_stub.go b/node/iptables_stub.go index 8f62c23..eeda907 100644 --- a/node/iptables_stub.go +++ b/node/iptables_stub.go @@ -8,6 +8,9 @@ import ( "fmt" ) -func setupIPTables(_ context.Context, _ string, _ []string) error { +func setupIPTables(_ context.Context, _ string, _ []string, _ bool, _ []string) error { return fmt.Errorf("iptables setup: %w", errors.ErrUnsupported) } + +// ClearDropRules is a no-op off Linux, where cocoon-net installs no rules. +func ClearDropRules(_ context.Context) error { return nil } diff --git a/node/node.go b/node/node.go index ab0ae1b..027bc85 100644 --- a/node/node.go +++ b/node/node.go @@ -32,6 +32,16 @@ type Config struct { // SkipIPTables omits the iptables FORWARD + NAT MASQUERADE rules. SkipIPTables bool + + // DropInternalAccess blocks VM-to-VM traffic within SubnetCIDR (cocoon's + // own range). Enforced as a FORWARD DROP; node setup enables the + // bridge-nf-call-iptables it relies on. Ignored when SkipIPTables is set. + DropInternalAccess bool + + // DropCIDRs lists additional external destination CIDRs VM traffic is + // blocked from reaching (e.g. management ranges). Same enforcement and + // SkipIPTables gating as DropInternalAccess. + DropCIDRs []string } // Setup configures host networking components (idempotent). @@ -53,7 +63,7 @@ func Setup(ctx context.Context, cfg *Config) error { if cfg.SkipIPTables { logger.Info(ctx, "iptables setup skipped (SkipIPTables=true)") - } else if err := setupIPTables(ctx, cfg.SubnetCIDR, cfg.SecondaryNICs); err != nil { + } else if err := setupIPTables(ctx, cfg.SubnetCIDR, cfg.SecondaryNICs, cfg.DropInternalAccess, cfg.DropCIDRs); err != nil { return fmt.Errorf("iptables: %w", err) } diff --git a/node/sysctl.go b/node/sysctl.go index 1241979..4e373c7 100644 --- a/node/sysctl.go +++ b/node/sysctl.go @@ -46,6 +46,10 @@ func setupSysctl(ctx context.Context, primaryNIC string, secondaryNICs []string) } func writeSysctl(key, val string) error { - path := filepath.Join(procSysBase, strings.ReplaceAll(key, ".", "/")) - return os.WriteFile(path, []byte(val), filePerm) //nolint:gosec // sysctl tuning + return os.WriteFile(sysctlPath(key), []byte(val), filePerm) //nolint:gosec // sysctl tuning +} + +// sysctlPath maps a dotted sysctl key to its /proc/sys file path. +func sysctlPath(key string) string { + return filepath.Join(procSysBase, strings.ReplaceAll(key, ".", "/")) } diff --git a/pool/pool.go b/pool/pool.go index e976dc3..2a0cb5f 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -48,6 +48,12 @@ type State struct { // before this field existed; daemon falls back to built-in defaults. DNSServers []string `json:"dnsServers,omitempty"` + // DropInternalAccess blocks VM-to-VM traffic within the subnet; DropCIDRs + // lists additional external ranges VM traffic is blocked from reaching. + // Both are enforced by node setup as FORWARD DROP rules. + DropInternalAccess bool `json:"dropInternalAccess,omitempty"` + DropCIDRs []string `json:"dropCIDRs,omitempty"` + // timestamps UpdatedAt time.Time `json:"updatedAt"` }