diff --git a/internal/validation/suite.go b/internal/validation/suite.go index 880ca7b..ad79d17 100644 --- a/internal/validation/suite.go +++ b/internal/validation/suite.go @@ -129,6 +129,16 @@ func RunInstanceLifecycleValidation(t *testing.T, config ProviderConfig) { require.NoError(t, err, "ValidateDockerFirewallBlocksPort should pass - docker port should be blocked by iptables") }) + t.Run("ValidateDockerFirewallAllowsEgress", func(t *testing.T) { + err := v1.ValidateDockerFirewallAllowsEgress(ctx, client, instance, ssh.GetTestPrivateKey()) + require.NoError(t, err, "ValidateDockerFirewallAllowsEgress should pass - egress should be allowed") + }) + + t.Run("ValidateDockerFirewallAllowsContainerToContainerCommunication", func(t *testing.T) { + err := v1.ValidateDockerFirewallAllowsContainerToContainerCommunication(ctx, client, instance, ssh.GetTestPrivateKey()) + require.NoError(t, err, "ValidateDockerFirewallAllowsContainerToContainerCommunication should pass - container to container communication should be allowed") + }) + if capabilities.IsCapable(v1.CapabilityStopStartInstance) && instance.Stoppable { t.Run("ValidateStopStartInstance", func(t *testing.T) { err := v1.ValidateStopStartInstance(ctx, client, instance) @@ -322,6 +332,16 @@ func RunFirewallValidation(t *testing.T, config ProviderConfig, opts FirewallVal require.NoError(t, err, "ValidateDockerFirewallBlocksPort should pass - docker port should be blocked") }) + t.Run("ValidateDockerFirewallAllowsEgress", func(t *testing.T) { + err := v1.ValidateDockerFirewallAllowsEgress(ctx, client, instance, ssh.GetTestPrivateKey()) + require.NoError(t, err, "ValidateDockerFirewallAllowsEgress should pass - egress should be allowed") + }) + + t.Run("ValidateDockerFirewallAllowsContainerToContainerCommunication", func(t *testing.T) { + err := v1.ValidateDockerFirewallAllowsContainerToContainerCommunication(ctx, client, instance, ssh.GetTestPrivateKey()) + require.NoError(t, err, "ValidateDockerFirewallAllowsContainerToContainerCommunication should pass - container to container communication should be allowed") + }) + // Test that SSH port is accessible (sanity check) t.Run("ValidateSSHPortAccessible", func(t *testing.T) { err := v1.ValidateFirewallAllowsPort(ctx, client, instance, ssh.GetTestPrivateKey(), instance.SSHPort) diff --git a/v1/networking_validation.go b/v1/networking_validation.go index 7cef7da..4fc4b8d 100644 --- a/v1/networking_validation.go +++ b/v1/networking_validation.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "strings" "time" "github.com/brevdev/cloud/internal/ssh" @@ -141,6 +142,144 @@ func ValidateDockerFirewallBlocksPort(ctx context.Context, client CloudInstanceR return nil } +func ValidateDockerFirewallAllowsEgress(ctx context.Context, client CloudInstanceReader, instance *Instance, privateKey string) error { + var err error + instance, err = WaitForInstanceLifecycleStatus(ctx, client, instance, LifecycleStatusRunning, PendingToRunningTimeout) + if err != nil { + return fmt.Errorf("failed to wait for instance running: %w", err) + } + + publicIP := instance.PublicIP + if publicIP == "" { + return fmt.Errorf("public IP is not available for instance %s", instance.CloudID) + } + + sshClient, err := ssh.ConnectToHost(ctx, ssh.ConnectionConfig{ + User: instance.SSHUser, + HostPort: fmt.Sprintf("%s:%d", publicIP, instance.SSHPort), + PrivKey: privateKey, + }) + if err != nil { + return fmt.Errorf("failed to SSH into instance: %w", err) + } + defer func() { _ = sshClient.Close() }() + + dockerCmd, err := setupDockerCommand(ctx, sshClient, instance.CloudID) + if err != nil { + return err + } + + // Pull the alpine image + cmd := fmt.Sprintf( + "%s pull alpine", + dockerCmd, + ) + _, stderr, err := sshClient.RunCommand(ctx, cmd) + if err != nil { + return fmt.Errorf("failed to pull alpine image: %w, stderr: %s", err, stderr) + } + + // Start a Docker container to ping Google's DNS server + cmd = fmt.Sprintf( + "%s run --rm alpine ping -c 3 8.8.8.8", + dockerCmd, + ) + stdout, stderr, err := sshClient.RunCommand(ctx, cmd) + if err != nil { + return fmt.Errorf("failed to connect to Google's DNS server: %w, stderr: %s", err, stderr) + } + if !strings.Contains(stdout, "3 packets transmitted, 3 packets received") { + return fmt.Errorf("expected successful ping, got: %s", stdout) + } + + return nil +} + +func ValidateDockerFirewallAllowsContainerToContainerCommunication(ctx context.Context, client CloudInstanceReader, instance *Instance, privateKey string) error { + var err error + instance, err = WaitForInstanceLifecycleStatus(ctx, client, instance, LifecycleStatusRunning, PendingToRunningTimeout) + if err != nil { + return fmt.Errorf("failed to wait for instance running: %w", err) + } + + publicIP := instance.PublicIP + if publicIP == "" { + return fmt.Errorf("public IP is not available for instance %s", instance.CloudID) + } + + sshClient, err := ssh.ConnectToHost(ctx, ssh.ConnectionConfig{ + User: instance.SSHUser, + HostPort: fmt.Sprintf("%s:%d", publicIP, instance.SSHPort), + PrivKey: privateKey, + }) + if err != nil { + return fmt.Errorf("failed to SSH into instance: %w", err) + } + defer func() { _ = sshClient.Close() }() + + dockerCmd, err := setupDockerCommand(ctx, sshClient, instance.CloudID) + if err != nil { + return err + } + + // Create a docker network + networkName := "firewall-test-network" + cmd := fmt.Sprintf( + "%s network create %s", + dockerCmd, networkName, + ) + _, stderr, err := sshClient.RunCommand(ctx, cmd) + if err != nil { + return fmt.Errorf("failed to create docker network: %w, stderr: %s", err, stderr) + } + + // Pull the alpine image + cmd = fmt.Sprintf( + "%s pull alpine", + dockerCmd, + ) + _, stderr, err = sshClient.RunCommand(ctx, cmd) + if err != nil { + return fmt.Errorf("failed to pull alpine image: %w, stderr: %s", err, stderr) + } + + // Pull the nginx image + cmd = fmt.Sprintf( + "%s pull nginx:alpine", + dockerCmd, + ) + _, stderr, err = sshClient.RunCommand(ctx, cmd) + if err != nil { + return fmt.Errorf("failed to pull nginx image: %w, stderr: %s", err, stderr) + } + + // Start a Docker container in the background + containerName := "firewall-test-container-to-container" + cmd = fmt.Sprintf( + "%s run -d --name %s --network %s nginx:alpine", + dockerCmd, containerName, networkName, + ) + _, stderr, err = sshClient.RunCommand(ctx, cmd) + if err != nil { + return fmt.Errorf("failed to start docker container: %w, stderr: %s", err, stderr) + } + + // Start a second Docker container to connect to the first container + cmd = fmt.Sprintf( + "%s run --network %s --rm alpine wget -q -O- http://%s", + dockerCmd, networkName, containerName, + ) + stdout, stderr, err := sshClient.RunCommand(ctx, cmd) + if err != nil { + return fmt.Errorf("failed to connect to nginx container: %w, stderr: %s", err, stderr) + } + + if !strings.Contains(stdout, "Welcome to nginx") { + return fmt.Errorf("expected successful wget, got: %s", stdout) + } + return nil +} + // setupDockerCommand ensures Docker is available and returns the command to use (always with sudo) func setupDockerCommand(ctx context.Context, sshClient *ssh.Client, instanceID CloudProviderInstanceID) (string, error) { // Check if Docker is available diff --git a/v1/providers/nebius/instance.go b/v1/providers/nebius/instance.go index 89bccae..6f43534 100644 --- a/v1/providers/nebius/instance.go +++ b/v1/providers/nebius/instance.go @@ -1810,6 +1810,10 @@ func generateIPTablesCommands() []string { commands := []string{ "iptables -F DOCKER-USER", "iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT", + "iptables -A DOCKER-USER -i docker0 ! -o docker0 -j ACCEPT", + "iptables -A DOCKER-USER -i br+ ! -o br+ -j ACCEPT", + "iptables -A DOCKER-USER -i docker0 -o docker0 -j ACCEPT", + "iptables -A DOCKER-USER -i br+ -o br+ -j ACCEPT", "iptables -A DOCKER-USER -i lo -j ACCEPT", "iptables -A DOCKER-USER -j DROP", "iptables -A DOCKER-USER -j RETURN", // Expected by Docker diff --git a/v1/providers/shadeform/firewall.go b/v1/providers/shadeform/firewall.go index 693a18c..13a0013 100644 --- a/v1/providers/shadeform/firewall.go +++ b/v1/providers/shadeform/firewall.go @@ -15,11 +15,26 @@ const ( ufwDefaultAllowPort2222 = "ufw allow 2222/tcp" ufwForceEnable = "ufw --force enable" - ipTablesResetDockerUserChain = "iptables -F DOCKER-USER" - ipTablesAllowDockerUserOutbound = "iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT" + // Clear DOCKER-USER policy. + ipTablesResetDockerUserChain = "iptables -F DOCKER-USER" + + // Allow return traffic. + ipTablesAllowDockerUserOutbound = "iptables -A DOCKER-USER -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT" + + // Allow containers to initiate outbound traffic (default bridge + user-defined bridges). + ipTablesAllowDockerUserOutboundInit0 = "iptables -A DOCKER-USER -i docker0 ! -o docker0 -j ACCEPT" + ipTablesAllowDockerUserOutboundInit1 = "iptables -A DOCKER-USER -i br+ ! -o br+ -j ACCEPT" + + // Allow container-to-container on the same bridge. + ipTablesAllowDockerUserDockerToDocker0 = "iptables -A DOCKER-USER -i docker0 -o docker0 -j ACCEPT" + ipTablesAllowDockerUserDockerToDocker1 = "iptables -A DOCKER-USER -i br+ -o br+ -j ACCEPT" + + // Allow inbound traffic on the loopback interface. ipTablesAllowDockerUserInpboundLoopback = "iptables -A DOCKER-USER -i lo -j ACCEPT" - ipTablesDropDockerUserInbound = "iptables -A DOCKER-USER -j DROP" - ipTablesReturnDockerUser = "iptables -A DOCKER-USER -j RETURN" + + // Drop everything else. + ipTablesDropDockerUserInbound = "iptables -A DOCKER-USER -j DROP" + ipTablesReturnDockerUser = "iptables -A DOCKER-USER -j RETURN" ) func (c *ShadeformClient) GenerateFirewallScript(firewallRules v1.FirewallRules) (string, error) { @@ -63,6 +78,10 @@ func (c *ShadeformClient) getIPTablesCommands() []string { commands := []string{ ipTablesResetDockerUserChain, ipTablesAllowDockerUserOutbound, + ipTablesAllowDockerUserOutboundInit0, + ipTablesAllowDockerUserOutboundInit1, + ipTablesAllowDockerUserDockerToDocker0, + ipTablesAllowDockerUserDockerToDocker1, ipTablesAllowDockerUserInpboundLoopback, ipTablesDropDockerUserInbound, ipTablesReturnDockerUser, // Expected by Docker