diff --git a/agent-vm.nix b/agent-vm.nix index 65f47bd..549e931 100644 --- a/agent-vm.nix +++ b/agent-vm.nix @@ -35,7 +35,7 @@ in LinkLocalAddressing = "no"; }; address = [ "${net.agentIp}/${toString net.networkPrefix}" ]; - routes = [ { Gateway = net.firewallIp; } ]; + routes = [ { Gateway = net.agentDefaultGatewayIp; } ]; dns = [ net.firewallIp ]; }; diff --git a/bun.lock b/bun.lock index 7e1ec37..39df676 100644 --- a/bun.lock +++ b/bun.lock @@ -5,7 +5,10 @@ "": { "name": "rootcell", "dependencies": { + "@aws-sdk/client-ec2": "^3.1050.0", + "@aws-sdk/client-s3": "^3.1050.0", "@aws-sdk/client-secrets-manager": "^3.1050.0", + "@aws-sdk/client-sts": "^3.1050.0", "@aws-sdk/credential-providers": "^3.1050.0", "yargs": "18.0.0", "zod": "^4.4.3", @@ -26,6 +29,10 @@ "packages": { "@aws-crypto/crc32": ["@aws-crypto/crc32@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg=="], + "@aws-crypto/crc32c": ["@aws-crypto/crc32c@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag=="], + + "@aws-crypto/sha1-browser": ["@aws-crypto/sha1-browser@5.2.0", "", { "dependencies": { "@aws-crypto/supports-web-crypto": "^5.2.0", "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg=="], + "@aws-crypto/sha256-browser": ["@aws-crypto/sha256-browser@5.2.0", "", { "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "@aws-crypto/supports-web-crypto": "^5.2.0", "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw=="], "@aws-crypto/sha256-js": ["@aws-crypto/sha256-js@5.2.0", "", { "dependencies": { "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "tslib": "^2.6.2" } }, "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA=="], @@ -36,10 +43,18 @@ "@aws-sdk/client-cognito-identity": ["@aws-sdk/client-cognito-identity@3.1050.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/credential-provider-node": "^3.972.43", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-VBbHeLRUoprJE57zc8WEbqC+bLUbHGlMI8S7NUuvEZ3svsYp0SGzkw81xuq2W7zbnlNFJL1jMIINjNMGo0Rn1Q=="], + "@aws-sdk/client-ec2": ["@aws-sdk/client-ec2@3.1050.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/credential-provider-node": "^3.972.43", "@aws-sdk/middleware-sdk-ec2": "^3.972.26", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-pJrYSya1WQs+2Wvvz8oC9bezZmbdOCFhRG39e1NIxyhAoMsHk2GQ7Yp0sKqjjn7IXKvaGAekIhdh+s3NTfK9Qg=="], + + "@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.1050.0", "", { "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/credential-provider-node": "^3.972.43", "@aws-sdk/middleware-bucket-endpoint": "^3.972.14", "@aws-sdk/middleware-expect-continue": "^3.972.12", "@aws-sdk/middleware-flexible-checksums": "^3.974.20", "@aws-sdk/middleware-location-constraint": "^3.972.10", "@aws-sdk/middleware-sdk-s3": "^3.972.41", "@aws-sdk/middleware-ssec": "^3.972.10", "@aws-sdk/signature-v4-multi-region": "^3.996.27", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-9kgtv+bXZQrOIJT2INPPBCezrJu1FlgGrzEat/ut4A4V53IT00LynsBZgp12eFKbjJuNCeTo7iPSKjPsX8ub+A=="], + "@aws-sdk/client-secrets-manager": ["@aws-sdk/client-secrets-manager@3.1050.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/credential-provider-node": "^3.972.43", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-mVOzxK4inCJgbGSQTeENYaACA9qGikru07b3HJyrKEeLVhuLKmo5csVzvKuz++ROdhX9gR7WzzJkoDOO5Xy/EQ=="], + "@aws-sdk/client-sts": ["@aws-sdk/client-sts@3.1050.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/credential-provider-node": "^3.972.43", "@aws-sdk/signature-v4-multi-region": "^3.996.27", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-CaQcTKGwxUEmU661gu3OxV9Ep7/jX39C83jfd/HnW/o7O/W9/bb+Jn1aMS05cdpst6wGBQ2kzgsCXGaBZGuLxQ=="], + "@aws-sdk/core": ["@aws-sdk/core@3.974.12", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@aws-sdk/xml-builder": "^3.972.24", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.2", "@smithy/signature-v4": "^5.4.2", "@smithy/types": "^4.14.1", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A=="], + "@aws-sdk/crc64-nvme": ["@aws-sdk/crc64-nvme@3.972.8", "", { "dependencies": { "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA=="], + "@aws-sdk/credential-provider-cognito-identity": ["@aws-sdk/credential-provider-cognito-identity@3.972.35", "", { "dependencies": { "@aws-sdk/nested-clients": "^3.997.10", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-mMQsBJv40oi5QdqRj4Xbc9jTlWMxqWfs5zWu+RhbOuF5F0AxxWXT70hm0abOmLbF2M/Tkuygs01H4eWIQMfoMw=="], "@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.38", "", { "dependencies": { "@aws-sdk/core": "^3.974.12", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w=="], @@ -60,6 +75,20 @@ "@aws-sdk/credential-providers": ["@aws-sdk/credential-providers@3.1050.0", "", { "dependencies": { "@aws-sdk/client-cognito-identity": "3.1050.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/credential-provider-cognito-identity": "^3.972.35", "@aws-sdk/credential-provider-env": "^3.972.38", "@aws-sdk/credential-provider-http": "^3.972.40", "@aws-sdk/credential-provider-ini": "^3.972.42", "@aws-sdk/credential-provider-login": "^3.972.42", "@aws-sdk/credential-provider-node": "^3.972.43", "@aws-sdk/credential-provider-process": "^3.972.38", "@aws-sdk/credential-provider-sso": "^3.972.42", "@aws-sdk/credential-provider-web-identity": "^3.972.42", "@aws-sdk/nested-clients": "^3.997.10", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/credential-provider-imds": "^4.3.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-IvGKl6+Vwf/x/wzgfWHjUwKu+0x5BeB7uwSO3QGH/9ssuAdpZR+maBUDP+HYbBgG2mU+CufN/eTelVwnWh10FQ=="], + "@aws-sdk/middleware-bucket-endpoint": ["@aws-sdk/middleware-bucket-endpoint@3.972.14", "", { "dependencies": { "@aws-sdk/core": "^3.974.12", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-Aaj0d+xbo1jJquBWJP0/9V/XZRYukO3LWIRp3dOLHmoFrYKb4YZ0aLefgVHfGcNOVBS2ZTq7L/n5JcrE7DaC+Q=="], + + "@aws-sdk/middleware-expect-continue": ["@aws-sdk/middleware-expect-continue@3.972.12", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-dA5pKTom/Ls9mgeyeaRBNQrRIVOLVjv4AmKOB0/e4yaiXEUy0gSz2d3liP8JHtYoCAEWySU1jWnyzwLOREN+4g=="], + + "@aws-sdk/middleware-flexible-checksums": ["@aws-sdk/middleware-flexible-checksums@3.974.20", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", "@aws-crypto/util": "5.2.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/crc64-nvme": "^3.972.8", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-NdnMVQCR1YjIcqFAiNLdBiOwr2DyQDB2IiXQrBhzolKOv32ae4d4Ll7IzLMi04eMHiq/o/Y/GjFuVjF9HuG0QA=="], + + "@aws-sdk/middleware-location-constraint": ["@aws-sdk/middleware-location-constraint@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ=="], + + "@aws-sdk/middleware-sdk-ec2": ["@aws-sdk/middleware-sdk-ec2@3.972.26", "", { "dependencies": { "@aws-sdk/core": "^3.974.12", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/signature-v4": "^5.4.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-sHc/vgigKtDZa1D19Go9jQT/IjMACinFnwg7I+vmhdie3rPFjB5VF57T9cDgcG0TAQnhBTkXSm1w4+ZlKR0bEA=="], + + "@aws-sdk/middleware-sdk-s3": ["@aws-sdk/middleware-sdk-s3@3.972.41", "", { "dependencies": { "@aws-sdk/core": "^3.974.12", "@aws-sdk/signature-v4-multi-region": "^3.996.27", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/signature-v4": "^5.4.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-M4T2I2WPuH5WQpU8Tsp+u2bcO29zGRkU14ATzuqb9I4xh8tzsLqtp4hzaJM5aO2dhMZnHDzyQwSFVgc3XbnoGg=="], + + "@aws-sdk/middleware-ssec": ["@aws-sdk/middleware-ssec@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw=="], + "@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.10", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.12", "@aws-sdk/signature-v4-multi-region": "^3.996.27", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/fetch-http-handler": "^5.4.2", "@smithy/node-http-handler": "^4.7.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg=="], "@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.27", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/signature-v4": "^5.4.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg=="], diff --git a/common.nix b/common.nix index 0ad5aa5..fbdc444 100644 --- a/common.nix +++ b/common.nix @@ -4,10 +4,16 @@ # that are genuinely VM-specific (hostname, networking, firewall policy, # services) live in agent-vm.nix and firewall-vm.nix respectively. +let + net = import ./network.nix; + isAwsEc2 = (net.provider or "lima") == "aws-ec2"; +in { imports = [ # Required for the guest to boot under virtio VM runtimes. (modulesPath + "/profiles/qemu-guest.nix") + ] ++ lib.optionals isAwsEc2 [ + (modulesPath + "/virtualisation/amazon-image.nix") ]; config = { @@ -42,13 +48,13 @@ options = "--delete-older-than 14d"; }; - boot.loader.grub = { + boot.loader.grub = lib.mkIf (!isAwsEc2) { device = lib.mkDefault "nodev"; efiSupport = lib.mkDefault true; efiInstallAsRemovable = lib.mkDefault true; }; - fileSystems."/boot" = { + fileSystems."/boot" = lib.mkIf (!isAwsEc2) { device = lib.mkDefault "/dev/vda1"; fsType = lib.mkDefault "vfat"; }; @@ -61,7 +67,7 @@ }; environment.enableAllTerminfo = true; - services.lima.enable = true; + services.lima.enable = !isAwsEc2; networking.nat.enable = lib.mkForce false; # Lima's hostagent probes `/bin/bash` even when the configured user shell is diff --git a/firewall-vm.nix b/firewall-vm.nix index 31377a2..2aff5b6 100644 --- a/firewall-vm.nix +++ b/firewall-vm.nix @@ -89,6 +89,7 @@ in systemd.network.networks."10-egress" = { matchConfig = egressMatch; networkConfig.DHCP = "ipv4"; + dns = net.firewallUpstreamDns; }; # Private Lima user-v2 link to the agent VM. diff --git a/network.nix b/network.nix index 5b90116..6000109 100644 --- a/network.nix +++ b/network.nix @@ -13,6 +13,8 @@ let defaults = { + provider = "lima"; + # IP of the firewall VM on the private inter-VM network. The agent VM uses # this as its default route, DNS server, and SSH proxy. # @@ -23,6 +25,11 @@ let # IP of the agent VM on the same private network. agentIp = "192.168.100.11"; + # Default gateway inside the agent VM. Lima uses the firewall's private IP. + # AWS uses the VPC subnet router, whose private route table sends default + # traffic to the firewall ENI. + agentDefaultGatewayIp = "192.168.100.10"; + # Subnet prefix length for the inter-VM network. networkPrefix = 24; @@ -31,6 +38,11 @@ let firewallPrivateInterface = "enp0s1"; firewallEgressInterface = "enp0s2"; firewallControlInterface = "enp0s2"; + + # Optional static upstream resolvers for the firewall VM. Lima normally + # gets this from DHCP. AWS disables VPC DNS for the rootcell VPC, so the + # firewall receives explicit public upstream resolvers instead. + firewallUpstreamDns = []; }; override = diff --git a/package.json b/package.json index 5c1bffe..884b43e 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,10 @@ "vitest": "^4.0.0" }, "dependencies": { + "@aws-sdk/client-ec2": "^3.1050.0", + "@aws-sdk/client-s3": "^3.1050.0", "@aws-sdk/client-secrets-manager": "^3.1050.0", + "@aws-sdk/client-sts": "^3.1050.0", "@aws-sdk/credential-providers": "^3.1050.0", "yargs": "18.0.0", "zod": "^4.4.3" diff --git a/src/rootcell/instance.ts b/src/rootcell/instance.ts index 454c0ec..ba6b9b6 100644 --- a/src/rootcell/instance.ts +++ b/src/rootcell/instance.ts @@ -131,7 +131,8 @@ function instanceHasVmState(repoDir: string, instanceName: string, env: NodeJS.P const paths = instancePaths(repoDir, instanceName, env); return existsSync(join(paths.dir, "v", "a")) || existsSync(join(paths.dir, "v", "f")) - || existsSync(join(paths.dir, "v", "n")); + || existsSync(join(paths.dir, "v", "n")) + || existsSync(join(paths.dir, "v", "aws-ec2")); } function rootcellInstanceFromPaths(paths: InstancePaths, state: InstanceState): RootcellInstance { diff --git a/src/rootcell/integration/common/assertions.ts b/src/rootcell/integration/common/assertions.ts index 900eeb1..71fdf81 100644 --- a/src/rootcell/integration/common/assertions.ts +++ b/src/rootcell/integration/common/assertions.ts @@ -31,7 +31,7 @@ export async function expectSpyWiring(flow: IntegrationFlow): Promise { export async function expectPrivateNetworkRouting(flow: IntegrationFlow): Promise { const network = flow.providers.network.plan().guest; await flow.agentSh(`ip -4 -o addr show ${network.agentPrivateInterface} | grep -q '${network.agentIp}/'`); - await flow.agentSh(`ip route show default | grep -q '^default via ${network.firewallIp} dev ${network.agentPrivateInterface}'`); + await flow.agentSh(`ip route show default | grep -q '^default via ${network.agentDefaultGatewayIp ?? network.firewallIp} dev ${network.agentPrivateInterface}'`); await flow.firewallSh(`ip -4 -o addr show ${network.firewallPrivateInterface} | grep -q '${network.firewallIp}/'`); await flow.agentSh(`dig @${network.firewallIp} +short +time=3 +tries=1 github.com | grep -qE '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$'`); } diff --git a/src/rootcell/integration/common/provider-spec.ts b/src/rootcell/integration/common/provider-spec.ts index ceda674..5dcd889 100644 --- a/src/rootcell/integration/common/provider-spec.ts +++ b/src/rootcell/integration/common/provider-spec.ts @@ -1,5 +1,6 @@ import type { RootcellConfig } from "../../types.ts"; import type { ProviderBundle, VmNetworkAttachment } from "../../providers/types.ts"; +import { awsEc2IntegrationProvider } from "../providers/aws-ec2/provider.ts"; import { macOsLimaUserV2IntegrationProvider } from "../providers/macos-lima-user-v2/provider.ts"; export interface IntegrationProviderSpec { @@ -14,6 +15,7 @@ export interface IntegrationProviderSpec = { + id: "aws-ec2", + platform: "darwin", + architecture: "arm64", + guestArchitecture: "aarch64-linux", + createBundle, + preflight: preflightAwsEc2Integration, + stopTestResources: stopAwsEc2TestResources, + removeTestState: removeAwsEc2TestState, +}; + +export function createBundle( + config: RootcellConfig, + log: (message: string) => void, +): ProviderBundle { + return { + network: new AwsEc2NetworkProvider(config, log), + vm: new AwsEc2VmProvider(config, log), + secrets: new StaticSecretProviderRegistry([ + new MacOsKeychainSecretProvider(), + ...config.awsSecretsManagerProviders.map((providerConfig) => new AwsSecretsManagerSecretProvider(providerConfig)), + ]), + }; +} + +export function preflightAwsEc2Integration(): Promise { + requireEnv("ROOTCELL_VM_PROVIDER", "aws-ec2"); + requireEnv("ROOTCELL_AWS_PROFILE"); + requireEnv("ROOTCELL_AWS_REGION"); + for (const tool of ["terraform", "ssh", "scp", "ssh-keygen", "curl"]) { + if (!commandExists(tool)) { + throw new Error(`aws-ec2 integration tests require '${tool}' on PATH`); + } + } + return Promise.resolve(); +} + +export async function stopAwsEc2TestResources(repoDir: string): Promise { + await stopAwsEc2InstanceResources(repoDir, TEST_INSTANCE); + await stopAwsEc2InstanceResources(repoDir, LIFECYCLE_INSTANCE); +} + +export async function removeAwsEc2TestState(repoDir: string): Promise { + await removeAwsEc2InstanceState(repoDir, TEST_INSTANCE); + await removeAwsEc2InstanceState(repoDir, LIFECYCLE_INSTANCE); +} + +function stopAwsEc2InstanceResources(repoDir: string, instance: string): Promise { + if (!existsSync(instancePaths(repoDir, instance, process.env).statePath)) { + return Promise.resolve(); + } + runInherited(join(repoDir, "rootcell"), ["stop", "--instance", instance], { + cwd: repoDir, + allowFailure: true, + }); + return Promise.resolve(); +} + +function removeAwsEc2InstanceState(repoDir: string, instance: string): Promise { + const paths = instancePaths(repoDir, instance, process.env); + if (!existsSync(paths.statePath)) { + return Promise.resolve(); + } + const result = runInherited(join(repoDir, "rootcell"), ["remove", "--instance", instance], { + cwd: repoDir, + allowFailure: true, + }); + if (result.status !== 0) { + throw new Error(`rootcell remove failed for AWS EC2 integration instance '${instance}'`); + } + rmSync(paths.dir, { recursive: true, force: true }); + return Promise.resolve(); +} + +function requireEnv(name: string, expected?: string): void { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`aws-ec2 integration tests require ${name}`); + } + if (expected !== undefined && value !== expected) { + throw new Error(`aws-ec2 integration tests require ${name}=${expected}`); + } +} diff --git a/src/rootcell/integration/providers/macos-lima-user-v2/provider.ts b/src/rootcell/integration/providers/macos-lima-user-v2/provider.ts index c7efa05..de7a455 100644 --- a/src/rootcell/integration/providers/macos-lima-user-v2/provider.ts +++ b/src/rootcell/integration/providers/macos-lima-user-v2/provider.ts @@ -145,6 +145,7 @@ function limaCleanupConfig(repoDir: string, instance: string, env: NodeJS.Proces networkPrefix: "24", imageManifestUrl: "https://example.invalid/manifest.json", awsSecretsManagerProviders: [], + vmProvider: "lima", }; } diff --git a/src/rootcell/providers/aws-ec2-aws.ts b/src/rootcell/providers/aws-ec2-aws.ts new file mode 100644 index 0000000..f69c09b --- /dev/null +++ b/src/rootcell/providers/aws-ec2-aws.ts @@ -0,0 +1,253 @@ +import { + DescribeInstancesCommand, + DescribeKeyPairsCommand, + DescribeTagsCommand, + EC2Client, + type EC2ClientConfig, + type DescribeInstancesCommandOutput, +} from "@aws-sdk/client-ec2"; +import { + DeleteObjectsCommand, + GetObjectTaggingCommand, + ListObjectsV2Command, + S3Client, + type S3ClientConfig, +} from "@aws-sdk/client-s3"; +import { + GetCallerIdentityCommand, + STSClient, + type STSClientConfig, +} from "@aws-sdk/client-sts"; +import { fromIni } from "@aws-sdk/credential-providers"; +import type { RootcellConfig } from "../types.ts"; + +export interface AwsEc2Api { + accountId(): Promise; + instanceStatus(instanceId: string): Promise<"missing" | "running" | "stopped" | "unexpected">; + assertTagged(resourceIds: readonly string[], requiredTags: Readonly>): Promise; + assertKeyPairsTagged(keyNames: readonly string[], requiredTags: Readonly>): Promise; + assertS3ObjectsTagged(objects: readonly AwsS3ObjectRef[], requiredTags: Readonly>): Promise; + deleteS3Prefix(bucket: string, prefix: string): Promise; +} + +export interface AwsS3ObjectRef { + readonly bucket: string; + readonly key: string; +} + +export class DefaultAwsEc2Api implements AwsEc2Api { + private readonly credentials: NonNullable; + private ec2: EC2Client | undefined; + private s3: S3Client | undefined; + private sts: STSClient | undefined; + + constructor(private readonly config: RootcellConfig) { + const aws = this.requireAwsConfig(); + this.credentials = fromIni({ profile: aws.profile }); + } + + async accountId(): Promise { + const response = await this.stsClient().send(new GetCallerIdentityCommand({})); + if (response.Account === undefined || response.Account.length === 0) { + throw new Error("AWS STS GetCallerIdentity returned no account id"); + } + return response.Account; + } + + async instanceStatus(instanceId: string): Promise<"missing" | "running" | "stopped" | "unexpected"> { + let response: DescribeInstancesCommandOutput; + try { + response = await this.ec2Client().send(new DescribeInstancesCommand({ + InstanceIds: [instanceId], + })); + } catch (error) { + if (awsErrorName(error) === "InvalidInstanceID.NotFound") { + return "missing"; + } + throw error; + } + const instance = response.Reservations?.flatMap((reservation) => reservation.Instances ?? [])[0]; + const state = instance?.State?.Name; + if (state === undefined) { + return "missing"; + } + if (state === "running" || state === "stopped") { + return state; + } + if (state === "terminated" || state === "shutting-down") { + return "missing"; + } + return "unexpected"; + } + + async assertTagged(resourceIds: readonly string[], requiredTags: Readonly>): Promise { + const ids = [...new Set(resourceIds)]; + for (const id of ids) { + const byResource = new Map>(); + let nextToken: string | undefined; + do { + const response = await this.ec2Client().send(new DescribeTagsCommand({ + Filters: [ + { Name: "resource-id", Values: [id] }, + ], + NextToken: nextToken, + })); + for (const tag of response.Tags ?? []) { + if (tag.ResourceId === undefined || tag.Key === undefined || tag.Value === undefined) { + continue; + } + const tags = byResource.get(tag.ResourceId) ?? new Map(); + tags.set(tag.Key, tag.Value); + byResource.set(tag.ResourceId, tags); + } + nextToken = response.NextToken; + } while (nextToken !== undefined); + const tags = byResource.get(id); + for (const [key, value] of Object.entries(requiredTags)) { + if (tags?.get(key) !== value) { + throw new Error(`refusing to delete AWS resource ${id}: missing required tag ${key}=${value}`); + } + } + } + } + + async assertKeyPairsTagged(keyNames: readonly string[], requiredTags: Readonly>): Promise { + for (const keyName of [...new Set(keyNames)]) { + const response = await this.ec2Client().send(new DescribeKeyPairsCommand({ + KeyNames: [keyName], + })); + const keyPair = response.KeyPairs?.find((candidate) => candidate.KeyName === keyName); + const tagPairs = (keyPair?.Tags ?? []).flatMap((tag): [string, string][] => { + if (tag.Key === undefined || tag.Value === undefined) { + return []; + } + return [[tag.Key, tag.Value]]; + }); + const tags = new Map(tagPairs); + for (const [key, value] of Object.entries(requiredTags)) { + if (tags.get(key) !== value) { + throw new Error(`refusing to delete AWS key pair ${keyName}: missing required tag ${key}=${value}`); + } + } + } + } + + async assertS3ObjectsTagged(objects: readonly AwsS3ObjectRef[], requiredTags: Readonly>): Promise { + for (const object of objects) { + if (!await this.s3ObjectHasRequiredTags(object.bucket, object.key, requiredTags)) { + throw new Error(`refusing to delete AWS S3 object s3://${object.bucket}/${object.key}: missing required rootcell tags`); + } + } + } + + async deleteS3Prefix(bucket: string, prefix: string): Promise { + let continuationToken: string | undefined; + do { + const listed = await this.s3Client().send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: prefix.endsWith("/") ? prefix : `${prefix}/`, + ContinuationToken: continuationToken, + })); + const objects = (listed.Contents ?? []) + .map((object) => object.Key) + .filter((key): key is string => key !== undefined && key.length > 0) + .filter((key) => this.keyIsInsidePrefix(key, prefix)); + const taggedObjects = []; + for (const key of objects) { + if (await this.s3ObjectHasRequiredTags(bucket, key, { + RootcellManaged: "true", + RootcellInstanceName: this.config.instanceName, + })) { + taggedObjects.push({ Key: key }); + } + } + if (taggedObjects.length > 0) { + await this.s3Client().send(new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { + Objects: taggedObjects, + Quiet: true, + }, + })); + } + continuationToken = listed.NextContinuationToken; + } while (continuationToken !== undefined); + } + + private async s3ObjectHasRequiredTags( + bucket: string, + key: string, + requiredTags: Readonly>, + ): Promise { + const response = await this.s3Client().send(new GetObjectTaggingCommand({ + Bucket: bucket, + Key: key, + })); + const pairs: [string, string][] = []; + for (const tag of response.TagSet ?? []) { + if (tag.Key !== undefined && tag.Value !== undefined) { + pairs.push([tag.Key, tag.Value]); + } + } + const tags = new Map(pairs); + return Object.entries(requiredTags).every(([keyName, value]) => tags.get(keyName) === value); + } + + private keyIsInsidePrefix(key: string, prefix: string): boolean { + const normalized = prefix.endsWith("/") ? prefix : `${prefix}/`; + return key.startsWith(normalized); + } + + private ec2Client(): EC2Client { + this.ec2 ??= new EC2Client(this.clientConfig()); + return this.ec2; + } + + private s3Client(): S3Client { + this.s3 ??= new S3Client(this.s3ClientConfig()); + return this.s3; + } + + private stsClient(): STSClient { + this.sts ??= new STSClient(this.stsClientConfig()); + return this.sts; + } + + private clientConfig(): EC2ClientConfig & { readonly region: string } { + const aws = this.requireAwsConfig(); + return { + region: aws.region, + credentials: this.credentials, + }; + } + + private s3ClientConfig(): S3ClientConfig & { readonly region: string } { + const aws = this.requireAwsConfig(); + return { + region: aws.region, + credentials: this.credentials, + }; + } + + private stsClientConfig(): STSClientConfig & { readonly region: string } { + const aws = this.requireAwsConfig(); + return { + region: aws.region, + credentials: this.credentials, + }; + } + + private requireAwsConfig(): NonNullable { + if (this.config.awsEc2 === undefined) { + throw new Error("AWS EC2 provider config is missing"); + } + return this.config.awsEc2; + } +} + +function awsErrorName(error: unknown): string { + if (error !== null && typeof error === "object" && "name" in error && typeof error.name === "string") { + return error.name; + } + return ""; +} diff --git a/src/rootcell/providers/aws-ec2-config.ts b/src/rootcell/providers/aws-ec2-config.ts new file mode 100644 index 0000000..c17f486 --- /dev/null +++ b/src/rootcell/providers/aws-ec2-config.ts @@ -0,0 +1,49 @@ +import { parseSchema } from "../schema.ts"; +import { AwsEc2ConfigSchema, RootcellVmProviderIdSchema, type AwsEc2Config, type RootcellVmProviderId } from "../types.ts"; + +export const ROOTCELL_VM_PROVIDER_ENV = "ROOTCELL_VM_PROVIDER"; +export const ROOTCELL_AWS_PROFILE_ENV = "ROOTCELL_AWS_PROFILE"; +export const ROOTCELL_AWS_REGION_ENV = "ROOTCELL_AWS_REGION"; +export const ROOTCELL_AWS_CONTROL_CIDR_ENV = "ROOTCELL_AWS_CONTROL_CIDR"; +const NIXOS_AMI_OWNER_ID = "427812963091"; + +export function parseRootcellVmProvider(env: NodeJS.ProcessEnv): RootcellVmProviderId { + const raw = env[ROOTCELL_VM_PROVIDER_ENV]; + if (raw === undefined || raw.length === 0) { + return "lima"; + } + return parseSchema(RootcellVmProviderIdSchema, raw, `invalid ${ROOTCELL_VM_PROVIDER_ENV}`); +} + +export function parseAwsEc2Config(env: NodeJS.ProcessEnv): AwsEc2Config { + return parseSchema(AwsEc2ConfigSchema, { + profile: requiredEnv(env, ROOTCELL_AWS_PROFILE_ENV), + region: requiredEnv(env, ROOTCELL_AWS_REGION_ENV), + controlCidr: env[ROOTCELL_AWS_CONTROL_CIDR_ENV] ?? "auto", + agentInstanceType: env.ROOTCELL_AWS_AGENT_INSTANCE_TYPE ?? "t4g.2xlarge", + firewallInstanceType: env.ROOTCELL_AWS_FIREWALL_INSTANCE_TYPE ?? "t4g.small", + agentRootVolumeGiB: positiveIntegerEnv(env, "ROOTCELL_AWS_AGENT_ROOT_VOLUME_GIB", 60), + firewallRootVolumeGiB: positiveIntegerEnv(env, "ROOTCELL_AWS_FIREWALL_ROOT_VOLUME_GIB", 16), + nixosAmiOwnerId: env.ROOTCELL_AWS_NIXOS_AMI_OWNER_ID ?? NIXOS_AMI_OWNER_ID, + nixosAmiNamePattern: env.ROOTCELL_AWS_NIXOS_AMI_NAME_PATTERN ?? "nixos/25.11*", + }, "invalid AWS EC2 provider config"); +} + +function requiredEnv(env: NodeJS.ProcessEnv, name: string): string { + const value = env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`${name} is required when ${ROOTCELL_VM_PROVIDER_ENV}=aws-ec2`); + } + return value; +} + +function positiveIntegerEnv(env: NodeJS.ProcessEnv, name: string, defaultValue: number): number { + const raw = env[name]; + if (raw === undefined || raw.length === 0) { + return defaultValue; + } + if (!/^[1-9][0-9]*$/.test(raw)) { + throw new Error(`${name} must be a positive integer`); + } + return Number(raw); +} diff --git a/src/rootcell/providers/aws-ec2-network.ts b/src/rootcell/providers/aws-ec2-network.ts new file mode 100644 index 0000000..3685f06 --- /dev/null +++ b/src/rootcell/providers/aws-ec2-network.ts @@ -0,0 +1,89 @@ +import type { RootcellConfig } from "../types.ts"; +import type { NetworkPlan, NetworkProvider, VmNetworkAttachment, VmRole } from "./types.ts"; +import { AwsEc2TerraformProject } from "./aws-ec2-terraform.ts"; + +export interface AwsEc2NetworkAttachment extends VmNetworkAttachment { + readonly kind: "aws-ec2"; + readonly role: VmRole; + readonly privateIp: string; + readonly privateInterface: string; + readonly hasPublicControlInterface: boolean; + readonly hasEgress: boolean; +} + +export class AwsEc2NetworkProvider implements NetworkProvider { + readonly id = "aws-ec2"; + private readonly terraform: AwsEc2TerraformProject; + + constructor( + private readonly config: RootcellConfig, + private readonly log: (message: string) => void, + ) { + this.terraform = new AwsEc2TerraformProject(config, log); + } + + plan(): NetworkPlan { + const agentPrivateInterface = "ens5"; + const firewallPrivateInterface = "ens6"; + const firewallEgressInterface = "ens5"; + return { + provider: this.id, + guest: { + firewallIp: this.config.firewallIp, + agentIp: this.config.agentIp, + agentDefaultGatewayIp: awsVpcRouterIp(this.config), + networkPrefix: 24, + agentPrivateInterface, + firewallPrivateInterface, + firewallEgressInterface, + firewallControlInterface: firewallEgressInterface, + firewallUpstreamDns: ["1.1.1.1", "8.8.8.8"], + }, + vms: { + agent: { + kind: "aws-ec2", + role: "agent", + privateInterface: agentPrivateInterface, + privateIp: this.config.agentIp, + hasPublicControlInterface: false, + hasEgress: false, + }, + firewall: { + kind: "aws-ec2", + role: "firewall", + privateInterface: firewallPrivateInterface, + privateIp: this.config.firewallIp, + hasPublicControlInterface: true, + hasEgress: true, + }, + }, + }; + } + + async preflight(): Promise { + await this.terraform.preflight(); + } + + stop(): Promise { + return Promise.resolve(); + } + + async remove(): Promise { + await this.terraform.verifyTerraformStateTags(); + this.terraform.destroy(); + this.terraform.removeLocalState(); + } + + async ensureReady(input: { + readonly affectedVms: readonly string[]; + readonly force?: boolean; + readonly stopVmIfRunning: (name: string) => Promise; + }): Promise { + await this.terraform.ensureApplied({ force: input.force === true }); + } +} + +export function awsVpcRouterIp(config: RootcellConfig): string { + const subnet = config.firewallIp.slice(0, config.firewallIp.lastIndexOf(".")); + return `${subnet}.1`; +} diff --git a/src/rootcell/providers/aws-ec2-terraform.ts b/src/rootcell/providers/aws-ec2-terraform.ts new file mode 100644 index 0000000..e3069fc --- /dev/null +++ b/src/rootcell/providers/aws-ec2-terraform.ts @@ -0,0 +1,853 @@ +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import { loadDotEnv } from "../env.ts"; +import { resolveHostTool } from "../host-tools.ts"; +import { runCapture, runInherited } from "../process.ts"; +import { NonEmptyStringSchema, parseSchema } from "../schema.ts"; +import type { RootcellConfig } from "../types.ts"; +import { DefaultAwsEc2Api } from "./aws-ec2-aws.ts"; +import type { AwsEc2Api } from "./aws-ec2-aws.ts"; + +const AWS_PROVIDER_VERSION = "~> 6.0"; +const METADATA_VERSION = 1; + +const AwsEc2TerraformOutputsSchema = z.object({ + agent_instance_id: z.string(), + firewall_instance_id: z.string(), + firewall_public_ip: z.string(), + agent_private_ip: z.string(), + firewall_private_ip: z.string(), + nixos_ami_id: z.string(), + nixos_ami_name: z.string(), + applied_control_cidr: z.string(), +}).strict(); + +export type AwsEc2TerraformOutputs = Readonly>; + +const AwsEc2ProviderMetadataSchema = z.object({ + schemaVersion: z.literal(METADATA_VERSION), + rootcellInstanceId: NonEmptyStringSchema, + accountId: NonEmptyStringSchema, + region: NonEmptyStringSchema, + terraformDir: NonEmptyStringSchema, + outputs: AwsEc2TerraformOutputsSchema.optional(), +}).strict(); + +export type AwsEc2ProviderMetadata = Readonly>; + +export interface TerraformRunner { + init(cwd: string, env: NodeJS.ProcessEnv): void; + apply(cwd: string, env: NodeJS.ProcessEnv): void; + destroy(cwd: string, env: NodeJS.ProcessEnv): void; + outputJson(cwd: string, env: NodeJS.ProcessEnv): string; +} + +interface EnsureAppliedOptions { + readonly force: boolean; +} + +type DesiredInstanceState = "running" | "stopped"; + +interface TerraformVars { + readonly aws_profile: string; + readonly aws_region: string; + readonly instance_name: string; + readonly rootcell_instance_id: string; + readonly vpc_cidr: string; + readonly firewall_private_ip: string; + readonly agent_private_ip: string; + readonly control_cidr: string; + readonly public_key_path: string; + readonly guest_user: string; + readonly desired_instance_state: DesiredInstanceState; + readonly agent_instance_type: string; + readonly firewall_instance_type: string; + readonly agent_root_volume_gib: number; + readonly firewall_root_volume_gib: number; + readonly nixos_ami_owner_id: string; + readonly nixos_ami_name_pattern: string; +} + +export class AwsEc2TerraformProject { + private terraformBin = ""; + private readonly runner: TerraformRunner; + private readonly api: AwsEc2Api; + + constructor( + private readonly config: RootcellConfig, + private readonly log: (message: string) => void, + options: { + readonly runner?: TerraformRunner; + readonly api?: AwsEc2Api; + } = {}, + ) { + this.runner = options.runner ?? new TerraformCliRunner(() => this.ensureTerraform()); + this.api = options.api ?? new DefaultAwsEc2Api(config); + } + + async preflight(): Promise { + this.requireAwsConfig(); + this.ensureTerraform(); + await this.api.accountId(); + } + + async ensureApplied(options: EnsureAppliedOptions): Promise { + const metadata = await this.ensureMetadata(); + const controlCidr = this.resolveControlCidr(); + if (metadata.outputs !== undefined && !options.force) { + if (metadata.outputs.applied_control_cidr !== controlCidr) { + throw new Error( + `AWS control CIDR is now ${controlCidr}, but this instance was applied with ${metadata.outputs.applied_control_cidr}; run rootcell --instance ${this.config.instanceName} provision to update firewall SSH ingress`, + ); + } + return; + } + + this.log(`applying AWS EC2 Terraform for instance '${this.config.instanceName}'...`); + this.writeTerraformFiles(metadata, "running", controlCidr); + this.runner.init(this.terraformDir(), this.terraformEnv()); + this.runner.apply(this.terraformDir(), this.terraformEnv()); + this.writeMetadata({ + ...metadata, + outputs: this.readOutputs(), + }); + } + + async applyDesiredInstanceState(state: DesiredInstanceState): Promise { + const metadata = await this.ensureMetadata(); + const controlCidr = metadata.outputs?.applied_control_cidr ?? this.resolveControlCidr(); + this.writeTerraformFiles(metadata, state, controlCidr); + this.runner.init(this.terraformDir(), this.terraformEnv()); + this.runner.apply(this.terraformDir(), this.terraformEnv()); + this.writeMetadata({ + ...metadata, + outputs: this.readOutputs(), + }); + } + + destroy(): void { + if (!existsSync(this.terraformStatePath())) { + return; + } + this.runner.init(this.terraformDir(), this.terraformEnv()); + this.runner.destroy(this.terraformDir(), this.terraformEnv()); + } + + async verifyTerraformStateTags(): Promise { + const resources = this.destructiveResourceIdsFromState(); + const tags = ownershipTags(this.config.instanceName); + if (resources.ec2ResourceIds.length > 0) { + await this.api.assertTagged(resources.ec2ResourceIds, tags); + } + if (resources.keyPairNames.length > 0) { + await this.api.assertKeyPairsTagged(resources.keyPairNames, tags); + } + } + + removeLocalState(): void { + rmSync(this.awsDir(), { recursive: true, force: true }); + } + + readOutputsIfPresent(): AwsEc2TerraformOutputs | null { + return this.readMetadata()?.outputs ?? null; + } + + readMetadata(): AwsEc2ProviderMetadata | null { + const path = this.metadataPath(); + if (!existsSync(path)) { + return null; + } + try { + const raw: unknown = JSON.parse(readFileSync(path, "utf8")); + return parseSchema(AwsEc2ProviderMetadataSchema, raw, "invalid AWS EC2 provider metadata"); + } catch { + return null; + } + } + + controlIdentityPath(): string { + return join(this.config.instanceDir, "ssh", "aws_control_ed25519"); + } + + knownHostsPath(): string { + return join(this.config.instanceDir, "ssh", "known_hosts"); + } + + async instanceStatus(instanceId: string): Promise<"missing" | "running" | "stopped" | "unexpected"> { + return await this.api.instanceStatus(instanceId); + } + + private async ensureMetadata(): Promise { + const existing = this.readMetadata(); + if (existing !== null) { + return existing; + } + const accountId = await this.api.accountId(); + const metadata: AwsEc2ProviderMetadata = { + schemaVersion: METADATA_VERSION, + rootcellInstanceId: randomUUID(), + accountId, + region: this.requireAwsConfig().region, + terraformDir: this.terraformDir(), + }; + this.writeMetadata(metadata); + return metadata; + } + + private writeMetadata(metadata: AwsEc2ProviderMetadata): void { + mkdirSync(this.awsDir(), { recursive: true, mode: 0o700 }); + writeFileSync(this.metadataPath(), `${JSON.stringify(metadata, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); + } + + private writeTerraformFiles( + metadata: AwsEc2ProviderMetadata, + desiredState: DesiredInstanceState, + controlCidr: string, + ): void { + const terraformDir = this.terraformDir(); + mkdirSync(terraformDir, { recursive: true, mode: 0o700 }); + const vars = this.terraformVars(metadata.rootcellInstanceId, desiredState, controlCidr); + writeFileSync(join(terraformDir, "main.tf"), awsEc2TerraformMain(), { encoding: "utf8", mode: 0o600 }); + writeFileSync(join(terraformDir, "variables.tf"), awsEc2TerraformVariables(), { encoding: "utf8", mode: 0o600 }); + writeFileSync(join(terraformDir, "outputs.tf"), awsEc2TerraformOutputs(), { encoding: "utf8", mode: 0o600 }); + writeFileSync(join(terraformDir, "terraform.tfvars.json"), `${JSON.stringify(vars, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); + } + + private terraformVars(rootcellInstanceId: string, desiredState: DesiredInstanceState, controlCidr: string): TerraformVars { + const aws = this.requireAwsConfig(); + const keys = this.ensureControlKey(); + return { + aws_profile: aws.profile, + aws_region: aws.region, + instance_name: this.config.instanceName, + rootcell_instance_id: rootcellInstanceId, + vpc_cidr: `${this.subnetPrefix()}.0/24`, + firewall_private_ip: this.config.firewallIp, + agent_private_ip: this.config.agentIp, + control_cidr: controlCidr, + public_key_path: keys.publicKeyPath, + guest_user: this.config.guestUser, + desired_instance_state: desiredState, + agent_instance_type: aws.agentInstanceType, + firewall_instance_type: aws.firewallInstanceType, + agent_root_volume_gib: aws.agentRootVolumeGiB, + firewall_root_volume_gib: aws.firewallRootVolumeGiB, + nixos_ami_owner_id: aws.nixosAmiOwnerId, + nixos_ami_name_pattern: aws.nixosAmiNamePattern, + }; + } + + private ensureControlKey(): { readonly privateKeyPath: string; readonly publicKeyPath: string } { + const sshDir = join(this.config.instanceDir, "ssh"); + mkdirSync(sshDir, { recursive: true, mode: 0o700 }); + const privateKeyPath = this.controlIdentityPath(); + const publicKeyPath = `${privateKeyPath}.pub`; + if (!existsSync(privateKeyPath) || !existsSync(publicKeyPath)) { + runInherited("ssh-keygen", [ + "-t", + "ed25519", + "-N", + "", + "-C", + `rootcell-${this.config.instanceName}`, + "-f", + privateKeyPath, + ], { ignoredOutput: true }); + } + return { privateKeyPath, publicKeyPath }; + } + + private readOutputs(): AwsEc2TerraformOutputs { + const raw = JSON.parse(this.runner.outputJson(this.terraformDir(), this.terraformEnv())) as unknown; + if (raw === null || typeof raw !== "object") { + throw new Error("invalid terraform output: expected object"); + } + const object = raw as Record; + const flattened: Record = {}; + for (const [key, value] of Object.entries(object)) { + if (value !== null && typeof value === "object" && "value" in value) { + flattened[key] = (value as { readonly value?: unknown }).value; + } + } + return parseSchema(AwsEc2TerraformOutputsSchema, flattened, "invalid AWS EC2 Terraform outputs"); + } + + private destructiveResourceIdsFromState(): { + readonly ec2ResourceIds: readonly string[]; + readonly keyPairNames: readonly string[]; + } { + const path = this.terraformStatePath(); + if (!existsSync(path)) { + return { ec2ResourceIds: [], keyPairNames: [] }; + } + const raw = JSON.parse(readFileSync(path, "utf8")) as unknown; + const state = TerraformStateSchema.safeParse(raw); + if (!state.success) { + throw new Error("invalid Terraform state; refusing to destroy AWS resources"); + } + const ec2ResourceIds: string[] = []; + const keyPairNames: string[] = []; + const taggable = new Set([ + "aws_instance", + "aws_network_interface", + "aws_eip", + "aws_vpc", + "aws_subnet", + "aws_internet_gateway", + "aws_route_table", + "aws_security_group", + "aws_vpc_security_group_ingress_rule", + "aws_vpc_security_group_egress_rule", + ]); + for (const resource of state.data.resources) { + if (resource.type === "aws_key_pair") { + for (const instance of resource.instances) { + const attributes = instance.attributes as Record; + const id = stringAttribute(attributes, "id"); + if (id !== null) { + keyPairNames.push(id); + } + } + continue; + } + if (!taggable.has(resource.type)) { + continue; + } + for (const instance of resource.instances) { + const attributes = instance.attributes as Record; + const id = stringAttribute(attributes, "id"); + if (id !== null) { + ec2ResourceIds.push(id); + } + if (resource.type === "aws_instance") { + ec2ResourceIds.push(...rootVolumeIds(attributes)); + } + } + } + return { ec2ResourceIds, keyPairNames }; + } + + private resolveControlCidr(): string { + const configured = this.requireAwsConfig().controlCidr; + if (configured !== "auto") { + return configured; + } + const result = runCapture("curl", ["-fsSL", "--connect-timeout", "5", "--max-time", "20", "https://checkip.amazonaws.com"]); + const ip = result.stdout.trim(); + if (!/^[0-9]+(?:\.[0-9]+){3}$/.test(ip)) { + throw new Error(`failed to resolve current public IPv4 for ${this.config.instanceName}: ${ip}`); + } + return `${ip}/32`; + } + + private terraformEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + loadDotEnv(this.config.envPath, env); + env.AWS_PROFILE = this.requireAwsConfig().profile; + env.AWS_REGION = this.requireAwsConfig().region; + env.AWS_DEFAULT_REGION = this.requireAwsConfig().region; + return env; + } + + private ensureTerraform(): string { + if (this.terraformBin.length === 0) { + this.terraformBin = resolveHostTool({ + name: "terraform", + envVar: "ROOTCELL_TERRAFORM", + purpose: "to manage rootcell AWS EC2 resources", + }); + } + return this.terraformBin; + } + + private requireAwsConfig(): NonNullable { + if (this.config.awsEc2 === undefined) { + throw new Error("AWS EC2 provider config is missing"); + } + return this.config.awsEc2; + } + + private awsDir(): string { + return join(this.config.instanceDir, "v", "aws-ec2"); + } + + private terraformDir(): string { + return join(this.awsDir(), "terraform"); + } + + private terraformStatePath(): string { + return join(this.terraformDir(), "terraform.tfstate"); + } + + private metadataPath(): string { + return join(this.awsDir(), "metadata.json"); + } + + private subnetPrefix(): string { + return this.config.firewallIp.slice(0, this.config.firewallIp.lastIndexOf(".")); + } + +} + +export class TerraformCliRunner implements TerraformRunner { + constructor(private readonly terraformBin: () => string) {} + + init(cwd: string, env: NodeJS.ProcessEnv): void { + runInherited(this.terraformBin(), ["init", "-input=false"], { cwd, env }); + } + + apply(cwd: string, env: NodeJS.ProcessEnv): void { + runInherited(this.terraformBin(), ["apply", "-input=false", "-auto-approve"], { cwd, env }); + } + + destroy(cwd: string, env: NodeJS.ProcessEnv): void { + runInherited(this.terraformBin(), ["destroy", "-input=false", "-auto-approve"], { cwd, env }); + } + + outputJson(cwd: string, env: NodeJS.ProcessEnv): string { + return runCapture(this.terraformBin(), ["output", "-json"], { cwd, env }).stdout; + } +} + +const TerraformStateSchema = z.looseObject({ + resources: z.array(z.looseObject({ + type: z.string(), + instances: z.array(z.looseObject({ + attributes: z.looseObject({ + id: z.unknown().optional(), + }), + })), + })), +}); + +function stringAttribute(attributes: Readonly>, key: string): string | null { + const value = attributes[key]; + return typeof value === "string" && value.length > 0 ? value : null; +} + +function rootVolumeIds(attributes: Readonly>): readonly string[] { + const rootBlockDevice = attributes.root_block_device; + if (!Array.isArray(rootBlockDevice)) { + return []; + } + const ids: string[] = []; + for (const block of rootBlockDevice) { + if (block === null || typeof block !== "object" || Array.isArray(block)) { + continue; + } + const volumeId = stringAttribute(block as Record, "volume_id"); + if (volumeId !== null) { + ids.push(volumeId); + } + } + return ids; +} + +export function ownershipTags(instanceName: string): Record { + return { + RootcellManaged: "true", + RootcellInstanceName: instanceName, + }; +} + +export function awsEc2TerraformMain(): string { + return `terraform { + required_version = ">= 1.5.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "${AWS_PROVIDER_VERSION}" + } + } +} + +provider "aws" { + profile = var.aws_profile + region = var.aws_region + + default_tags { + tags = local.rootcell_tags + } +} + +locals { + rootcell_tags = { + RootcellManaged = "true" + RootcellInstanceName = var.instance_name + } + ssh_public_key = trimspace(file(var.public_key_path)) + rootcell_bootstrap_user_data = <<-ROOTCELL_USER_DATA +#!/bin/sh +set -eu +PATH=/run/current-system/sw/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +user=\${var.guest_user} +home=/home/\${var.guest_user} + +if ! getent group users >/dev/null 2>&1; then + groupadd -r users +fi + +if ! id -u "$user" >/dev/null 2>&1; then + useradd -m -u 501 -g users -G wheel -s /run/current-system/sw/bin/bash "$user" +else + usermod -a -G wheel "$user" + mkdir -p "$home" + chown "$user:users" "$home" +fi + +install -d -m 0700 -o "$user" -g users "$home/.ssh" +cat > "$home/.ssh/authorized_keys" <<'ROOTCELL_AUTHORIZED_KEYS' +\${local.ssh_public_key} +ROOTCELL_AUTHORIZED_KEYS +chown "$user:users" "$home/.ssh/authorized_keys" +chmod 0600 "$home/.ssh/authorized_keys" + +if ! grep -q "^$user .*NOPASSWD" /etc/sudoers; then + printf '%s ALL=(ALL:ALL) NOPASSWD: SETENV: ALL\\n' "$user" >> /etc/sudoers +fi +ROOTCELL_USER_DATA +} + +data "aws_ami" "nixos_arm64" { + owners = [var.nixos_ami_owner_id] + most_recent = true + + filter { + name = "name" + values = [var.nixos_ami_name_pattern] + } + + filter { + name = "architecture" + values = ["arm64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +resource "aws_vpc" "this" { + cidr_block = var.vpc_cidr + enable_dns_support = false + enable_dns_hostnames = false + tags = local.rootcell_tags +} + +resource "aws_subnet" "private" { + vpc_id = aws_vpc.this.id + cidr_block = cidrsubnet(var.vpc_cidr, 1, 0) + availability_zone = data.aws_availability_zones.available.names[0] + tags = local.rootcell_tags +} + +resource "aws_subnet" "public" { + vpc_id = aws_vpc.this.id + cidr_block = cidrsubnet(var.vpc_cidr, 1, 1) + availability_zone = data.aws_availability_zones.available.names[0] + map_public_ip_on_launch = false + tags = local.rootcell_tags +} + +data "aws_availability_zones" "available" { + state = "available" +} + +resource "aws_internet_gateway" "this" { + vpc_id = aws_vpc.this.id + tags = local.rootcell_tags +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.this.id + tags = local.rootcell_tags +} + +resource "aws_route" "public_default" { + route_table_id = aws_route_table.public.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.this.id +} + +resource "aws_route_table_association" "public" { + subnet_id = aws_subnet.public.id + route_table_id = aws_route_table.public.id +} + +resource "aws_route_table" "private" { + vpc_id = aws_vpc.this.id + tags = local.rootcell_tags +} + +resource "aws_route" "private_default" { + route_table_id = aws_route_table.private.id + destination_cidr_block = "0.0.0.0/0" + network_interface_id = aws_network_interface.firewall_private.id +} + +resource "aws_route_table_association" "private" { + subnet_id = aws_subnet.private.id + route_table_id = aws_route_table.private.id +} + +resource "aws_security_group" "firewall_public" { + name_prefix = "rootcell-\${var.instance_name}-firewall-public-" + description = "rootcell firewall public SSH" + vpc_id = aws_vpc.this.id + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_ingress_rule" "firewall_public_ssh" { + security_group_id = aws_security_group.firewall_public.id + cidr_ipv4 = var.control_cidr + from_port = 22 + ip_protocol = "tcp" + to_port = 22 + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_egress_rule" "firewall_public_all" { + security_group_id = aws_security_group.firewall_public.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" + tags = local.rootcell_tags +} + +resource "aws_security_group" "firewall_private" { + name_prefix = "rootcell-\${var.instance_name}-firewall-private-" + description = "rootcell firewall private policy ingress" + vpc_id = aws_vpc.this.id + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_ingress_rule" "firewall_private_https" { + security_group_id = aws_security_group.firewall_private.id + referenced_security_group_id = aws_security_group.agent.id + from_port = 443 + ip_protocol = "tcp" + to_port = 443 + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_ingress_rule" "firewall_private_connect" { + security_group_id = aws_security_group.firewall_private.id + referenced_security_group_id = aws_security_group.agent.id + from_port = 8080 + ip_protocol = "tcp" + to_port = 8081 + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_ingress_rule" "firewall_private_dns" { + security_group_id = aws_security_group.firewall_private.id + referenced_security_group_id = aws_security_group.agent.id + from_port = 53 + ip_protocol = "udp" + to_port = 53 + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_egress_rule" "firewall_private_all" { + security_group_id = aws_security_group.firewall_private.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" + tags = local.rootcell_tags +} + +resource "aws_security_group" "agent" { + name_prefix = "rootcell-\${var.instance_name}-agent-" + description = "rootcell agent private-only network" + vpc_id = aws_vpc.this.id + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_ingress_rule" "agent_ssh_from_firewall" { + security_group_id = aws_security_group.agent.id + referenced_security_group_id = aws_security_group.firewall_private.id + from_port = 22 + ip_protocol = "tcp" + to_port = 22 + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_egress_rule" "agent_https" { + security_group_id = aws_security_group.agent.id + cidr_ipv4 = "0.0.0.0/0" + from_port = 443 + ip_protocol = "tcp" + to_port = 443 + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_egress_rule" "agent_connect" { + security_group_id = aws_security_group.agent.id + cidr_ipv4 = "\${var.firewall_private_ip}/32" + from_port = 8080 + ip_protocol = "tcp" + to_port = 8080 + tags = local.rootcell_tags +} + +resource "aws_vpc_security_group_egress_rule" "agent_dns" { + security_group_id = aws_security_group.agent.id + cidr_ipv4 = "\${var.firewall_private_ip}/32" + from_port = 53 + ip_protocol = "udp" + to_port = 53 + tags = local.rootcell_tags +} + +resource "aws_key_pair" "control" { + key_name_prefix = "rootcell-\${var.instance_name}-" + public_key = file(var.public_key_path) + tags = local.rootcell_tags +} + +resource "aws_network_interface" "firewall_public" { + subnet_id = aws_subnet.public.id + security_groups = [aws_security_group.firewall_public.id] + tags = local.rootcell_tags +} + +resource "aws_network_interface" "firewall_private" { + subnet_id = aws_subnet.private.id + private_ips = [var.firewall_private_ip] + security_groups = [aws_security_group.firewall_private.id] + source_dest_check = false + tags = local.rootcell_tags +} + +resource "aws_network_interface" "agent" { + subnet_id = aws_subnet.private.id + private_ips = [var.agent_private_ip] + security_groups = [aws_security_group.agent.id] + tags = local.rootcell_tags +} + +resource "aws_eip" "firewall" { + domain = "vpc" + tags = local.rootcell_tags +} + +resource "aws_eip_association" "firewall" { + allocation_id = aws_eip.firewall.id + network_interface_id = aws_network_interface.firewall_public.id +} + +resource "aws_instance" "firewall" { + ami = data.aws_ami.nixos_arm64.id + instance_type = var.firewall_instance_type + key_name = aws_key_pair.control.key_name + user_data = local.rootcell_bootstrap_user_data + + network_interface { + network_interface_id = aws_network_interface.firewall_public.id + device_index = 0 + } + + network_interface { + network_interface_id = aws_network_interface.firewall_private.id + device_index = 1 + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + instance_metadata_tags = "disabled" + } + + root_block_device { + volume_size = var.firewall_root_volume_gib + volume_type = "gp3" + tags = local.rootcell_tags + } + + tags = local.rootcell_tags +} + +resource "aws_instance" "agent" { + ami = data.aws_ami.nixos_arm64.id + instance_type = var.agent_instance_type + key_name = aws_key_pair.control.key_name + user_data = local.rootcell_bootstrap_user_data + + network_interface { + network_interface_id = aws_network_interface.agent.id + device_index = 0 + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + instance_metadata_tags = "disabled" + } + + root_block_device { + volume_size = var.agent_root_volume_gib + volume_type = "gp3" + tags = local.rootcell_tags + } + + tags = local.rootcell_tags +} + +resource "aws_ec2_instance_state" "firewall" { + instance_id = aws_instance.firewall.id + state = var.desired_instance_state +} + +resource "aws_ec2_instance_state" "agent" { + instance_id = aws_instance.agent.id + state = var.desired_instance_state +} +`; +} + +export function awsEc2TerraformVariables(): string { + return `variable "aws_profile" { type = string } +variable "aws_region" { type = string } +variable "instance_name" { type = string } +variable "rootcell_instance_id" { type = string } +variable "vpc_cidr" { type = string } +variable "firewall_private_ip" { type = string } +variable "agent_private_ip" { type = string } +variable "control_cidr" { type = string } +variable "public_key_path" { type = string } +variable "guest_user" { type = string } +variable "desired_instance_state" { + type = string + validation { + condition = contains(["running", "stopped"], var.desired_instance_state) + error_message = "desired_instance_state must be running or stopped." + } +} +variable "agent_instance_type" { type = string } +variable "firewall_instance_type" { type = string } +variable "agent_root_volume_gib" { type = number } +variable "firewall_root_volume_gib" { type = number } +variable "nixos_ami_owner_id" { type = string } +variable "nixos_ami_name_pattern" { type = string } +`; +} + +export function awsEc2TerraformOutputs(): string { + return `output "agent_instance_id" { value = aws_instance.agent.id } +output "firewall_instance_id" { value = aws_instance.firewall.id } +output "firewall_public_ip" { value = aws_eip.firewall.public_ip } +output "agent_private_ip" { value = var.agent_private_ip } +output "firewall_private_ip" { value = var.firewall_private_ip } +output "nixos_ami_id" { value = data.aws_ami.nixos_arm64.id } +output "nixos_ami_name" { value = data.aws_ami.nixos_arm64.name } +output "applied_control_cidr" { value = var.control_cidr } +`; +} diff --git a/src/rootcell/providers/aws-ec2.ts b/src/rootcell/providers/aws-ec2.ts new file mode 100644 index 0000000..d0a60a1 --- /dev/null +++ b/src/rootcell/providers/aws-ec2.ts @@ -0,0 +1,153 @@ +import { ProxyJumpSshTransport, type ProxyJumpSshEndpoints } from "../transports/proxyjump-ssh.ts"; +import type { CommandResult, InheritedCommandResult } from "../types.ts"; +import type { RootcellConfig } from "../types.ts"; +import type { CopyToGuestOptions, ExecOptions, VmProvider, VmRole, VmStatus } from "./types.ts"; +import type { AwsEc2NetworkAttachment } from "./aws-ec2-network.ts"; +import { AwsEc2TerraformProject } from "./aws-ec2-terraform.ts"; + +export class AwsEc2VmProvider implements VmProvider { + readonly id = "aws-ec2"; + private readonly terraform: AwsEc2TerraformProject; + private readonly transport: ProxyJumpSshTransport; + + constructor( + private readonly config: RootcellConfig, + log: (message: string) => void, + ) { + this.terraform = new AwsEc2TerraformProject(config, log); + this.transport = new ProxyJumpSshTransport(config, () => this.transportEndpoints()); + } + + async status(name: string): Promise { + const outputs = this.terraform.readOutputsIfPresent(); + if (outputs === null) { + return { state: "missing" }; + } + const instanceId = this.instanceIdForName(name, outputs); + const state = await this.terraform.instanceStatus(instanceId); + if (state === "running" || state === "stopped" || state === "missing") { + return { state }; + } + return { state: "unexpected", detail: "unexpected AWS EC2 instance state" }; + } + + async forceStopIfRunning(name: string): Promise { + if ((await this.status(name)).state !== "running") { + return; + } + await this.terraform.applyDesiredInstanceState("stopped"); + } + + remove(): Promise { + return Promise.resolve(); + } + + assertCompatible(name: string, network: AwsEc2NetworkAttachment): Promise { + if (name === this.config.agentVm && network.role !== "agent") { + throw new Error(`${name} has incompatible AWS EC2 role metadata`); + } + if (name === this.config.firewallVm && network.role !== "firewall") { + throw new Error(`${name} has incompatible AWS EC2 role metadata`); + } + return Promise.resolve(); + } + + async ensureRunning(input: { + readonly role: VmRole; + readonly name: string; + readonly network: AwsEc2NetworkAttachment; + }): Promise<{ readonly created: boolean }> { + const status = await this.status(input.name); + if (status.state === "running") { + await this.waitForSsh(input.name); + return { created: false }; + } + if (status.state === "stopped") { + await this.terraform.applyDesiredInstanceState("running"); + await this.waitForSsh(input.name); + return { created: false }; + } + if (status.state === "missing") { + await this.terraform.ensureApplied({ force: true }); + await this.waitForSsh(input.name); + return { created: true }; + } + throw new Error(`${input.name} VM in unexpected state: ${status.detail}`); + } + + finalizeNetworking(): Promise { + return Promise.resolve(); + } + + forgetSshHostKey(name: string): Promise { + this.transport.forgetHostKey(name); + return Promise.resolve(); + } + + exec(name: string, command: readonly string[], options: ExecOptions = {}): Promise { + return this.transport.exec(name, command, options); + } + + execCapture(name: string, command: readonly string[], options: ExecOptions = {}): Promise { + return this.transport.execCapture(name, command, options); + } + + async execInteractive(name: string, command: readonly string[], options: ExecOptions = {}): Promise { + return await this.transport.execInteractive(name, command, options); + } + + copyToGuest(name: string, hostPath: string, guestPath: string, options: CopyToGuestOptions = {}): Promise { + return this.transport.copyToGuest(name, hostPath, guestPath, options); + } + + private transportEndpoints(): ProxyJumpSshEndpoints { + const outputs = this.terraform.readOutputsIfPresent(); + if (outputs === null) { + throw new Error("AWS EC2 Terraform outputs are not available yet"); + } + return { + firewallHost: outputs.firewall_public_ip, + firewallPort: 22, + agentHost: outputs.agent_private_ip, + identityPath: this.terraform.controlIdentityPath(), + knownHostsPath: this.terraform.knownHostsPath(), + }; + } + + private instanceIdForName(name: string, outputs: { + readonly agent_instance_id: string; + readonly firewall_instance_id: string; + }): string { + if (name === this.config.agentVm) { + return outputs.agent_instance_id; + } + if (name === this.config.firewallVm) { + return outputs.firewall_instance_id; + } + throw new Error(`unknown rootcell VM for AWS EC2 provider: ${name}`); + } + + private async waitForSsh(name: string): Promise { + let lastError = ""; + for (let attempt = 0; attempt < 300; attempt += 1) { + const result = await this.transport.execCapture(name, ["true"], { + allowFailure: true, + }); + if (result.status === 0) { + return; + } + const message = `${result.stderr}${result.stdout}`.trim(); + if (message.length > 0) { + lastError = message; + } + await sleep(1000); + } + throw new Error(`timeout waiting for SSH transport to ${name}${lastError.length === 0 ? "" : `: ${lastError}`}`); + } +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolveSleep) => { + setTimeout(resolveSleep, milliseconds); + }); +} diff --git a/src/rootcell/providers/aws-ec2/README.md b/src/rootcell/providers/aws-ec2/README.md new file mode 100644 index 0000000..7c7d6c2 --- /dev/null +++ b/src/rootcell/providers/aws-ec2/README.md @@ -0,0 +1,130 @@ +# AWS EC2 Provider + +The `aws-ec2` provider runs rootcell's agent and firewall VMs as EC2 +instances. Rootcell creates a dedicated VPC per rootcell instance and manages +AWS infrastructure with a generated Terraform module. + +## Required Instance Environment + +Set these in the instance `.env` before first use: + +```sh +ROOTCELL_VM_PROVIDER=aws-ec2 +ROOTCELL_AWS_PROFILE=your-profile +ROOTCELL_AWS_REGION=us-east-1 +ROOTCELL_AWS_CONTROL_CIDR=auto +``` + +`ROOTCELL_AWS_PROFILE` and `ROOTCELL_AWS_REGION` are required. Rootcell does +not fall back to `AWS_PROFILE` or `AWS_REGION` for provider selection. + +`ROOTCELL_AWS_CONTROL_CIDR=auto` resolves your current public IPv4 address to a +single `/32` when Terraform is applied. If that address changes, normal +`rootcell` entry fails with instructions to run `rootcell provision` so the +firewall SSH ingress rule is updated intentionally. + +## Terraform Layout + +Rootcell writes one Terraform module per instance: + +```text +/v/aws-ec2/ + metadata.json + terraform/ + main.tf + variables.tf + outputs.tf + terraform.tfvars.json + terraform.tfstate + .terraform.lock.hcl +``` + +Terraform state is the ownership record for AWS resources. Normal VM entry does +not run `terraform init` or `terraform apply`; it reads cached Terraform +outputs, checks EC2 status, syncs allowlists, injects explicitly configured +session secrets, and opens SSH through the firewall. + +Terraform runs for first create, explicit `rootcell provision`, Terraform-backed +start/stop transitions, and `rootcell remove`. + +## Upstream NixOS AMI + +AWS EC2 instances boot from the official upstream NixOS ARM64 AMI. Rootcell +does not use rootcell-owned release image manifests, VM Import/Export, imported +snapshots, generated AMIs, or S3 image staging. + +Terraform resolves the AMI at apply time: + +```hcl +data "aws_ami" "nixos_arm64" { + owners = [var.nixos_ami_owner_id] + most_recent = true + + filter { + name = "name" + values = [var.nixos_ami_name_pattern] + } + + filter { + name = "architecture" + values = ["arm64"] + } +} +``` + +The default owner is the official NixOS AMI publisher account +`427812963091`, and the default name pattern is `nixos/25.11*`. Override them +only when intentionally testing a different upstream image stream: + +```sh +ROOTCELL_AWS_NIXOS_AMI_OWNER_ID=427812963091 +ROOTCELL_AWS_NIXOS_AMI_NAME_PATTERN='nixos/25.11*' +``` + +Official NixOS AMIs initially accept SSH as `root`. Rootcell supplies a +non-secret EC2 user-data script containing only the generated SSH public key to +create the normal `luser` account before rootcell connects. No credentials or +secrets are placed in user data. + +## Ownership Tags + +Every AWS resource rootcell creates must have these tags where the AWS API +supports tagging: + +```text +RootcellManaged=true +RootcellInstanceName= +``` + +This includes VPC resources, security groups, key pairs, ENIs, EIPs, EC2 +instances, and root EBS volumes. The upstream NixOS AMI is not created by +rootcell and is not tagged or removed by rootcell. `rootcell remove` refuses to +delete recorded AWS resources that do not have both tags with the expected +values. + +## IAM And Credential Isolation + +Rootcell does not attach an IAM instance profile to the agent or firewall. The +generated Terraform module must not create IAM roles, IAM instance profiles, or +instance-profile associations for those instances. + +Rootcell never copies host `~/.aws` files into either VM and never injects AWS +credentials unless the user explicitly maps them in `secrets.env`. + +IMDS remains enabled for diagnostics such as instance ID, AZ, and network +metadata, but it is hardened: + +```hcl +metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + instance_metadata_tags = "disabled" +} +``` + +Do not put secrets in user data. Allowlisting AWS SDK endpoints grants network +reachability only; without explicitly injected credentials, the agent should not +be able to call same-account AWS APIs. Integration tests verify that +`aws sts get-caller-identity` fails inside the agent unless credentials were +explicitly injected. diff --git a/src/rootcell/providers/factory.ts b/src/rootcell/providers/factory.ts index a917e38..ebbd845 100644 --- a/src/rootcell/providers/factory.ts +++ b/src/rootcell/providers/factory.ts @@ -1,7 +1,9 @@ import type { RootcellConfig } from "../types.ts"; import type { ProviderBundle } from "./types.ts"; +import { AwsEc2VmProvider } from "./aws-ec2.ts"; +import { AwsEc2NetworkProvider } from "./aws-ec2-network.ts"; import { LimaVmProvider } from "./lima.ts"; -import { MacOsLimaUserV2NetworkProvider, type LimaUserV2NetworkAttachment } from "./macos-lima-user-v2-network.ts"; +import { MacOsLimaUserV2NetworkProvider } from "./macos-lima-user-v2-network.ts"; import { AwsSecretsManagerSecretProvider } from "../secrets/aws-secrets-manager.ts"; import { MacOsKeychainSecretProvider } from "../secrets/macos-keychain.ts"; import { StaticSecretProviderRegistry } from "../secrets/registry.ts"; @@ -9,7 +11,17 @@ import { StaticSecretProviderRegistry } from "../secrets/registry.ts"; export function createProviderBundle( config: RootcellConfig, log: (message: string) => void, -): ProviderBundle { +): ProviderBundle { + if (config.vmProvider === "aws-ec2") { + return { + network: new AwsEc2NetworkProvider(config, log), + vm: new AwsEc2VmProvider(config, log), + secrets: new StaticSecretProviderRegistry([ + new MacOsKeychainSecretProvider(), + ...config.awsSecretsManagerProviders.map((providerConfig) => new AwsSecretsManagerSecretProvider(providerConfig)), + ]), + }; + } return { network: new MacOsLimaUserV2NetworkProvider(config, log), vm: new LimaVmProvider(config, log), diff --git a/src/rootcell/providers/types.ts b/src/rootcell/providers/types.ts index 81ee747..07605ba 100644 --- a/src/rootcell/providers/types.ts +++ b/src/rootcell/providers/types.ts @@ -12,11 +12,13 @@ export type VmStatus = export interface GuestNetworkConfig { readonly firewallIp: string; readonly agentIp: string; + readonly agentDefaultGatewayIp?: string; readonly networkPrefix: 24; readonly agentPrivateInterface: string; readonly firewallPrivateInterface: string; readonly firewallEgressInterface: string; readonly firewallControlInterface?: string; + readonly firewallUpstreamDns?: readonly string[]; } export interface VmNetworkAttachment { @@ -37,6 +39,7 @@ export interface NetworkProvider; ensureReady(input: { readonly affectedVms: readonly string[]; + readonly force?: boolean; readonly stopVmIfRunning: (name: string) => Promise; }): Promise; } diff --git a/src/rootcell/rootcell.test.ts b/src/rootcell/rootcell.test.ts index e4d7083..a1def7d 100644 --- a/src/rootcell/rootcell.test.ts +++ b/src/rootcell/rootcell.test.ts @@ -7,6 +7,15 @@ import { resolveHostTool } from "./host-tools.ts"; import { buildConfig, formatVmList } from "./rootcell.ts"; import { deriveVmNames, instancePaths, listRootcellVmInstanceNames, loadRootcellInstance, seedRootcellInstanceFiles } from "./instance.ts"; import { runCapture } from "./process.ts"; +import { parseAwsEc2Config } from "./providers/aws-ec2-config.ts"; +import { AwsEc2NetworkProvider, awsVpcRouterIp } from "./providers/aws-ec2-network.ts"; +import { + AwsEc2TerraformProject, + awsEc2TerraformMain, + ownershipTags, + type TerraformRunner, +} from "./providers/aws-ec2-terraform.ts"; +import type { AwsEc2Api, AwsS3ObjectRef } from "./providers/aws-ec2-aws.ts"; import { createProviderBundle } from "./providers/factory.ts"; import { limaNetworkListIncludes, @@ -275,6 +284,61 @@ describe("environment parsing", () => { { id: "aws-dev", awsProfile: "dev" }, ]); }); + + test("requires explicit AWS EC2 provider profile and region", () => { + expect(() => buildConfig("/repo", { + ROOTCELL_VM_PROVIDER: "aws-ec2", + ROOTCELL_AWS_PROFILE: "dev", + }, fakeInstance("dev"))).toThrow("ROOTCELL_AWS_REGION is required"); + + const config = buildConfig("/repo", { + ROOTCELL_VM_PROVIDER: "aws-ec2", + ROOTCELL_AWS_PROFILE: "dev", + ROOTCELL_AWS_REGION: "us-east-1", + }, fakeInstance("dev")); + + expect(config.vmProvider).toBe("aws-ec2"); + expect(config.awsEc2).toEqual({ + profile: "dev", + region: "us-east-1", + controlCidr: "auto", + agentInstanceType: "t4g.2xlarge", + firewallInstanceType: "t4g.small", + agentRootVolumeGiB: 60, + firewallRootVolumeGiB: 16, + nixosAmiOwnerId: "427812963091", + nixosAmiNamePattern: "nixos/25.11*", + }); + }); + + test("parses AWS EC2 provider overrides", () => { + expect(parseAwsEc2Config({ + ROOTCELL_AWS_PROFILE: "prod", + ROOTCELL_AWS_REGION: "us-west-2", + ROOTCELL_AWS_CONTROL_CIDR: "198.51.100.10/32", + ROOTCELL_AWS_AGENT_INSTANCE_TYPE: "t4g.xlarge", + ROOTCELL_AWS_FIREWALL_INSTANCE_TYPE: "t4g.nano", + ROOTCELL_AWS_AGENT_ROOT_VOLUME_GIB: "80", + ROOTCELL_AWS_FIREWALL_ROOT_VOLUME_GIB: "20", + ROOTCELL_AWS_NIXOS_AMI_OWNER_ID: "123456789012", + ROOTCELL_AWS_NIXOS_AMI_NAME_PATTERN: "nixos/unstable*", + })).toEqual({ + profile: "prod", + region: "us-west-2", + controlCidr: "198.51.100.10/32", + agentInstanceType: "t4g.xlarge", + firewallInstanceType: "t4g.nano", + agentRootVolumeGiB: 80, + firewallRootVolumeGiB: 20, + nixosAmiOwnerId: "123456789012", + nixosAmiNamePattern: "nixos/unstable*", + }); + expect(() => parseAwsEc2Config({ + ROOTCELL_AWS_PROFILE: "prod", + ROOTCELL_AWS_REGION: "us-west-2", + ROOTCELL_AWS_CONTROL_CIDR: "999.51.100.10/32", + })).toThrow("IPv4 CIDR"); + }); }); describe("host tool resolution", () => { @@ -538,6 +602,13 @@ describe("VM and network providers", () => { expect(providers.secrets.ids).toEqual(["macos-keychain"]); }); + test("factory selects AWS EC2 providers from instance environment", () => { + const providers = createProviderBundle(buildConfig("/repo", awsEc2Env(), fakeInstance("dev")), ignoreLog); + expect(providers.network.id).toBe("aws-ec2"); + expect(providers.vm.id).toBe("aws-ec2"); + expect(providers.secrets.ids).toEqual(["macos-keychain"]); + }); + test("factory registers configured AWS Secrets Manager providers", () => { const config = buildConfig("/repo", { [AWS_SECRETS_MANAGER_PROVIDERS_ENV]: JSON.stringify({ @@ -609,6 +680,33 @@ describe("VM and network providers", () => { expect(plan.vms.agent.reservedIps).toEqual(["192.168.109.2", "192.168.109.3"]); }); + test("AWS EC2 provider exposes public firewall and private-only agent attachments", () => { + const config = buildConfig("/repo", awsEc2Env(), fakeInstance("dev")); + const plan = new AwsEc2NetworkProvider(config, ignoreLog).plan(); + expect(plan.provider).toBe("aws-ec2"); + expect(plan.guest).toEqual({ + firewallIp: "192.168.109.10", + agentIp: "192.168.109.11", + agentDefaultGatewayIp: "192.168.109.1", + networkPrefix: 24, + agentPrivateInterface: "ens5", + firewallPrivateInterface: "ens6", + firewallEgressInterface: "ens5", + firewallControlInterface: "ens5", + firewallUpstreamDns: ["1.1.1.1", "8.8.8.8"], + }); + expect(plan.vms.agent).toEqual({ + kind: "aws-ec2", + role: "agent", + privateInterface: "ens5", + privateIp: "192.168.109.11", + hasPublicControlInterface: false, + hasEgress: false, + }); + expect(plan.vms.firewall.hasPublicControlInterface).toBe(true); + expect(awsVpcRouterIp(config)).toBe("192.168.109.1"); + }); + test("user-v2 network plan reserves Lima gateway and DNS IPs", () => { const config = buildConfig("/repo", {}, fakeInstance("dev")); expect(limaUserV2ReservedIps(config)).toEqual({ @@ -717,7 +815,8 @@ describe("VM and network providers", () => { expect(commonModule).toContain("uid = lib.mkDefault 501;"); expect(commonModule).toContain('home = lib.mkDefault "/home/${username}";'); expect(commonModule).toContain("ln -sfn /run/current-system/sw/bin/bash /bin/bash"); - expect(commonModule).toContain("services.lima.enable = true;"); + expect(commonModule).toContain("services.lima.enable = !isAwsEc2;"); + expect(commonModule).toContain("/virtualisation/amazon-image.nix"); expect(commonModule).toContain("networking.nat.enable = lib.mkForce false;"); const firewallModule = readFileSync("firewall-vm.nix", "utf8"); @@ -738,6 +837,157 @@ describe("VM and network providers", () => { expect(script).toContain("! ip -4 -o addr show scope global | grep -v"); }); + test("generated AWS EC2 Terraform keeps IAM, IMDS, tagging, and networking invariants", () => { + const hcl = awsEc2TerraformMain(); + expect(hcl).toContain('RootcellManaged = "true"'); + expect(hcl).toContain("RootcellInstanceName = var.instance_name"); + expect(hcl).toContain('enable_dns_support = false'); + expect(hcl).toContain('enable_dns_hostnames = false'); + expect(hcl).toContain('http_tokens = "required"'); + expect(hcl).toContain("http_put_response_hop_limit = 1"); + expect(hcl).toContain('instance_metadata_tags = "disabled"'); + expect(hcl).toContain("source_dest_check = false"); + expect(hcl).toContain("network_interface_id = aws_network_interface.firewall_private.id"); + expect(hcl).toContain("resource \"aws_ec2_instance_state\" \"agent\""); + expect(hcl).toContain("data \"aws_ami\" \"nixos_arm64\""); + expect(hcl).toContain('values = ["arm64"]'); + expect(hcl).toContain("user_data = local.rootcell_bootstrap_user_data"); + expect(hcl).not.toContain("aws_s3_object"); + expect(hcl).not.toContain("aws_ebs_snapshot_import"); + expect(hcl).not.toContain("iam_instance_profile"); + expect(hcl).not.toContain("aws_iam_role"); + expect(hcl).not.toContain("aws_iam_instance_profile"); + expect(hcl).not.toMatch(/RootcellProvider|RootcellInstanceId/); + }); + + test("AWS ownership tags are limited to managed flag and instance name", () => { + expect(ownershipTags("dev")).toEqual({ + RootcellManaged: "true", + RootcellInstanceName: "dev", + }); + }); + + test("AWS EC2 remove verifies Terraform state tags for EC2 resources", async () => { + const repo = makeInstanceRepo(); + try { + const env = { + ...instanceEnv(repo), + ...awsEc2Env(), + }; + const config = buildConfig(repo, env, fakeInstance("dev", repo, env)); + const terraformDir = join(config.instanceDir, "v", "aws-ec2", "terraform"); + mkdirSync(terraformDir, { recursive: true }); + writeFileSync(join(terraformDir, "terraform.tfstate"), `${JSON.stringify({ + resources: [ + { + type: "aws_instance", + instances: [{ + attributes: { + id: "i-agent", + root_block_device: [{ volume_id: "vol-agent-root" }], + }, + }], + }, + { + type: "aws_route", + instances: [{ attributes: { id: "ignored-route" } }], + }, + { + type: "aws_key_pair", + instances: [{ attributes: { id: "rootcell-dev-key" } }], + }, + ], + }, null, 2)}\n`, "utf8"); + const api = new FakeAwsEc2Api(); + const project = new AwsEc2TerraformProject(config, ignoreLog, { + runner: new RecordingTerraformRunner(), + api, + }); + + await project.verifyTerraformStateTags(); + + expect(api.taggedResourceIds).toEqual(["i-agent", "vol-agent-root"]); + expect(api.taggedKeyPairs).toEqual(["rootcell-dev-key"]); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("normal AWS EC2 ensureReady reads metadata without applying Terraform", async () => { + const repo = makeInstanceRepo(); + try { + const config = buildConfig(repo, { + ...instanceEnv(repo), + ...awsEc2Env(), + ROOTCELL_AWS_CONTROL_CIDR: "198.51.100.10/32", + }, fakeInstance("dev", repo, instanceEnv(repo))); + const awsDir = join(config.instanceDir, "v", "aws-ec2"); + mkdirSync(awsDir, { recursive: true }); + writeFileSync(join(awsDir, "metadata.json"), `${JSON.stringify({ + schemaVersion: 1, + rootcellInstanceId: "instance-a1b2", + accountId: "123456789012", + region: "us-east-1", + terraformDir: join(awsDir, "terraform"), + outputs: fakeAwsOutputs("198.51.100.10/32"), + }, null, 2)}\n`, "utf8"); + const runner = new RecordingTerraformRunner(); + const project = new AwsEc2TerraformProject(config, ignoreLog, { + runner, + api: new FakeAwsEc2Api(), + }); + + await project.ensureApplied({ force: false }); + + expect(runner.calls).toEqual([]); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("normal AWS EC2 entry reports auto-control CIDR drift without applying Terraform", async () => { + const repo = makeInstanceRepo(); + try { + const env = { + ...instanceEnv(repo), + ...awsEc2Env(), + ROOTCELL_AWS_CONTROL_CIDR: "198.51.100.20/32", + }; + const config = buildConfig(repo, env, fakeInstance("dev", repo, env)); + const awsDir = join(config.instanceDir, "v", "aws-ec2"); + mkdirSync(awsDir, { recursive: true }); + writeFileSync(join(awsDir, "metadata.json"), `${JSON.stringify({ + schemaVersion: 1, + rootcellInstanceId: "instance-a1b2", + accountId: "123456789012", + region: "us-east-1", + terraformDir: join(awsDir, "terraform"), + outputs: fakeAwsOutputs("198.51.100.10/32"), + }, null, 2)}\n`, "utf8"); + const runner = new RecordingTerraformRunner(); + const project = new AwsEc2TerraformProject(config, ignoreLog, { + runner, + api: new FakeAwsEc2Api(), + }); + + await expect(project.ensureApplied({ force: false })).rejects.toThrow("run rootcell --instance dev provision"); + expect(runner.calls).toEqual([]); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("AWS EC2 README documents Terraform layout, upstream AMI, tags, and credential isolation", () => { + const readme = readFileSync("src/rootcell/providers/aws-ec2/README.md", "utf8"); + expect(readme).toContain("/v/aws-ec2/"); + expect(readme).toContain("official upstream NixOS ARM64 AMI"); + expect(readme).toContain("427812963091"); + expect(readme).toContain("RootcellManaged=true"); + expect(readme).toContain("RootcellInstanceName="); + expect(readme).toContain("does not attach an IAM instance profile"); + expect(readme).toContain("http_tokens"); + }); + test("detects existing Lima networks from limactl JSON output", () => { expect(limaNetworkListIncludes(JSON.stringify([{ name: "rootcell-123456abcdef" }]), "rootcell-123456abcdef")).toBe(true); expect(limaNetworkListIncludes(JSON.stringify([{ Name: "other" }]), "rootcell-123456abcdef")).toBe(false); @@ -1202,6 +1452,14 @@ function instanceEnv(repo: string): NodeJS.ProcessEnv { return { ROOTCELL_STATE_DIR: join(repo, ".state") }; } +function awsEc2Env(): NodeJS.ProcessEnv { + return { + ROOTCELL_VM_PROVIDER: "aws-ec2", + ROOTCELL_AWS_PROFILE: "dev", + ROOTCELL_AWS_REGION: "us-east-1", + }; +} + function makeInstanceRepo(): string { const repo = mkdtempSync(join(tmpdir(), "rootcell-instance-test-")); mkdirSync(join(repo, "proxy"), { recursive: true }); @@ -1247,3 +1505,72 @@ function fakeManifest(): Record { ], }; } + +function fakeAwsOutputs(controlCidr: string): Record { + return { + agent_instance_id: "i-agent", + firewall_instance_id: "i-firewall", + firewall_public_ip: "203.0.113.10", + agent_private_ip: "192.168.109.11", + firewall_private_ip: "192.168.109.10", + nixos_ami_id: "ami-nixos", + nixos_ami_name: "nixos/25.11-aarch64-linux", + applied_control_cidr: controlCidr, + }; +} + +class RecordingTerraformRunner implements TerraformRunner { + readonly calls: string[] = []; + + init(): void { + this.calls.push("init"); + } + + apply(): void { + this.calls.push("apply"); + } + + destroy(): void { + this.calls.push("destroy"); + } + + outputJson(): string { + this.calls.push("output"); + return JSON.stringify(Object.fromEntries( + Object.entries(fakeAwsOutputs("198.51.100.10/32")).map(([key, value]) => [key, { value }]), + )); + } +} + +class FakeAwsEc2Api implements AwsEc2Api { + readonly taggedResourceIds: string[] = []; + readonly taggedS3Objects: AwsS3ObjectRef[] = []; + readonly taggedKeyPairs: string[] = []; + + accountId(): Promise { + return Promise.resolve("123456789012"); + } + + instanceStatus(): Promise<"missing" | "running" | "stopped" | "unexpected"> { + return Promise.resolve("running"); + } + + assertTagged(resourceIds: readonly string[]): Promise { + this.taggedResourceIds.push(...resourceIds); + return Promise.resolve(); + } + + assertKeyPairsTagged(keyNames: readonly string[]): Promise { + this.taggedKeyPairs.push(...keyNames); + return Promise.resolve(); + } + + assertS3ObjectsTagged(objects: readonly AwsS3ObjectRef[]): Promise { + this.taggedS3Objects.push(...objects); + return Promise.resolve(); + } + + deleteS3Prefix(): Promise { + return Promise.resolve(); + } +} diff --git a/src/rootcell/rootcell.ts b/src/rootcell/rootcell.ts index 40ac1ff..1416908 100644 --- a/src/rootcell/rootcell.ts +++ b/src/rootcell/rootcell.ts @@ -19,6 +19,7 @@ import { seedRootcellInstanceFiles, } from "./instance.ts"; import { commandExists, runCapture, runInherited } from "./process.ts"; +import { parseAwsEc2Config, parseRootcellVmProvider } from "./providers/aws-ec2-config.ts"; import { createProviderBundle } from "./providers/factory.ts"; import type { NetworkPlan, ProviderBundle, VmNetworkAttachment, VmStatus } from "./providers/types.ts"; import { parseSchema } from "./schema.ts"; @@ -81,6 +82,10 @@ function shellQuote(value: string): string { return `'${value.replaceAll("'", "'\\''")}'`; } +function nixStringList(values: readonly string[]): string { + return `[ ${values.map(nixString).join(" ")} ]`; +} + function repoDirFromImportMeta(importMetaPath: string): string { let dir = dirname(resolve(importMetaPath)); for (;;) { @@ -97,6 +102,7 @@ function repoDirFromImportMeta(importMetaPath: string): string { export function buildConfig(repoDir: string, env: NodeJS.ProcessEnv, instance: RootcellInstance): RootcellConfig { const vmNames = deriveVmNames(instance.name); + const vmProvider = parseRootcellVmProvider(env); return parseSchema(RootcellConfigSchema, { repoDir, instanceName: instance.name, @@ -116,6 +122,8 @@ export function buildConfig(repoDir: string, env: NodeJS.ProcessEnv, instance: R imageManifestUrl: env.ROOTCELL_IMAGE_MANIFEST_URL ?? DEFAULT_IMAGE_MANIFEST_URL, ...(env.ROOTCELL_IMAGE_DIR === undefined || env.ROOTCELL_IMAGE_DIR.length === 0 ? {} : { imageDir: env.ROOTCELL_IMAGE_DIR }), awsSecretsManagerProviders: parseAwsSecretsManagerProviderConfigs(env), + vmProvider, + ...(vmProvider === "aws-ec2" ? { awsEc2: parseAwsEc2Config(env) } : {}), }, `invalid rootcell config for ${instance.name}`); } @@ -167,6 +175,7 @@ export class RootcellApp { await this.providers.network.ensureReady({ affectedVms: [this.config.agentVm, this.config.firewallVm], + force: subcommand === "provision", stopVmIfRunning: async (name) => { await this.providers.vm.forceStopIfRunning(name); }, @@ -192,7 +201,7 @@ export class RootcellApp { allowFailure: true, env: [ ...injectedSecretEnv, - `AWS_REGION=${process.env.AWS_REGION ?? "us-east-1"}`, + `AWS_REGION=${this.config.awsEc2?.region ?? process.env.AWS_REGION ?? "us-east-1"}`, "NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt", "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt", "REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt", @@ -258,8 +267,10 @@ export class RootcellApp { const content = [ "# Generated by ./rootcell from this instance's state. DO NOT EDIT.", "{", + ` provider = ${nixString(this.networkPlan.provider)};`, ` firewallIp = ${nixString(network.firewallIp)};`, ` agentIp = ${nixString(network.agentIp)};`, + ` agentDefaultGatewayIp = ${nixString(network.agentDefaultGatewayIp ?? network.firewallIp)};`, ` networkPrefix = ${String(network.networkPrefix)};`, ` agentPrivateInterface = ${nixString(network.agentPrivateInterface)};`, ` firewallPrivateInterface = ${nixString(network.firewallPrivateInterface)};`, @@ -267,6 +278,9 @@ export class RootcellApp { ...(network.firewallControlInterface === undefined ? [] : [ ` firewallControlInterface = ${nixString(network.firewallControlInterface)};`, ]), + ...(network.firewallUpstreamDns === undefined ? [] : [ + ` firewallUpstreamDns = ${nixStringList(network.firewallUpstreamDns)};`, + ]), "}", "", ].join("\n"); @@ -306,7 +320,7 @@ systemctl stop dhcpcd.service 2>/dev/null || true ip link set "$iface" up ip addr flush dev "$iface" ip addr add '${network.agentIp}/${String(network.networkPrefix)}' dev "$iface" -ip route replace default via '${network.firewallIp}' dev "$iface" +ip route replace default via '${network.agentDefaultGatewayIp ?? network.firewallIp}' dev "$iface" if command -v resolvectl >/dev/null 2>&1; then resolvectl dns "$iface" '${network.firewallIp}' || true resolvectl domain "$iface" '~.' || true @@ -316,6 +330,19 @@ printf 'nameserver %s\\n' '${network.firewallIp}' > /etc/resolv.conf await this.providers.vm.exec(this.config.agentVm, ["sudo", "bash", "-lc", script]); } + private async bootstrapFirewallDns(): Promise { + const nameservers = this.networkPlan.guest.firewallUpstreamDns ?? []; + if (nameservers.length === 0) { + return; + } + const resolvConf = `${nameservers.map((nameserver) => `nameserver ${nameserver}`).join("\n")}\n`; + await this.providers.vm.exec(this.config.firewallVm, [ + "bash", + "-lc", + `printf %s ${shellQuote(resolvConf)} | sudo tee /etc/resolv.conf >/dev/null`, + ]); + } + private async bootstrapAgentFirewallTrust(): Promise { const script = ` set -euo pipefail @@ -654,6 +681,7 @@ systemctl is-active mitmproxy-explicit >/dev/null 2>&1 \\ this.ensureCa(); await this.copyRepoIntoVm(this.config.firewallVm, VM_FILES.firewall); await this.copyGeneratedNetworkIntoVm(this.config.firewallVm); + await this.bootstrapFirewallDns(); await this.runNixosSwitch("firewall", ` set -e cd '${this.config.guestRepoDir}' @@ -750,7 +778,7 @@ Run \`./rootcell pubkey\` to print it again. private async runNixosSwitch(role: "agent" | "firewall", script: string): Promise { const name = role === "agent" ? this.config.agentVm : this.config.firewallVm; const network = role === "agent" ? this.networkPlan.vms.agent : this.networkPlan.vms.firewall; - const result = await this.providers.vm.exec(name, ["bash", "-lc", script], { + const result = await this.providers.vm.exec(name, ["bash", "-lc", `${this.bootPartitionBootstrapScript()}\n${script}`], { allowFailure: true, }); if (result.status !== 0 && result.status !== 255) { @@ -763,6 +791,15 @@ Run \`./rootcell pubkey\` to print it again. await this.providers.vm.finalizeNetworking?.({ role, name, network }); } + private bootPartitionBootstrapScript(): string { + return ` +if [ -e /dev/disk/by-label/ESP ] && ! findmnt -rn /boot >/dev/null 2>&1; then + sudo mkdir -p /boot + sudo mount /dev/disk/by-label/ESP /boot +fi +`; + } + private async runAgentHomeManager(): Promise { const script = ` set -e @@ -840,6 +877,12 @@ function appForInstance(repoDir: string, env: NodeJS.ProcessEnv, instance: Rootc return new RootcellApp(config, createProviderBundle(config, log)); } +function envForExistingInstance(repoDir: string, baseEnv: NodeJS.ProcessEnv, instanceName: string): NodeJS.ProcessEnv { + const env = { ...baseEnv }; + loadDotEnv(instancePaths(repoDir, instanceName, env).envPath, env); + return env; +} + async function runListCommand( repoDir: string, env: NodeJS.ProcessEnv, @@ -847,20 +890,22 @@ async function runListCommand( explicitInstance: boolean, ): Promise { if (explicitInstance) { - const instance = loadExistingRootcellInstance(repoDir, instanceName, env); + const instanceEnv = envForExistingInstance(repoDir, env, instanceName); + const instance = loadExistingRootcellInstance(repoDir, instanceName, instanceEnv); if (instance === null) { process.stdout.write(formatVmList(missingVmEntries(instanceName))); return 0; } - process.stdout.write(formatVmList(await appForInstance(repoDir, env, instance).listVms())); + process.stdout.write(formatVmList(await appForInstance(repoDir, instanceEnv, instance).listVms())); return 0; } const entries: VmListEntry[] = []; for (const name of listRootcellVmInstanceNames(repoDir, env)) { - const instance = loadExistingRootcellInstance(repoDir, name, env); + const instanceEnv = envForExistingInstance(repoDir, env, name); + const instance = loadExistingRootcellInstance(repoDir, name, instanceEnv); if (instance !== null) { - entries.push(...await appForInstance(repoDir, env, instance).listVms()); + entries.push(...await appForInstance(repoDir, instanceEnv, instance).listVms()); } } process.stdout.write(formatVmList(entries)); @@ -873,12 +918,13 @@ async function runLifecycleCommand( command: "stop" | "remove", instanceName: string, ): Promise { - const instance = loadExistingRootcellInstance(repoDir, instanceName, env); + const instanceEnv = envForExistingInstance(repoDir, env, instanceName); + const instance = loadExistingRootcellInstance(repoDir, instanceName, instanceEnv); if (instance === null) { log(`rootcell instance '${instanceName}' not found; run ./rootcell --instance ${instanceName} first.`); return 1; } - const app = appForInstance(repoDir, env, instance); + const app = appForInstance(repoDir, instanceEnv, instance); if (command === "stop") { await app.stopVms(); process.stdout.write(`stopped ${instanceName}\n`); diff --git a/src/rootcell/types.ts b/src/rootcell/types.ts index b78b6fc..c2871a5 100644 --- a/src/rootcell/types.ts +++ b/src/rootcell/types.ts @@ -7,6 +7,49 @@ import { } from "./schema.ts"; import { AwsSecretsManagerSecretProviderConfigSchema } from "./secrets/aws-secrets-manager-config.ts"; +export const RootcellVmProviderIdSchema = z.enum(["lima", "aws-ec2"]); + +export type RootcellVmProviderId = z.infer; + +const AwsControlCidrSchema = z.string().refine( + (value) => value === "auto" || isIpv4Cidr(value), + { message: "must be 'auto' or an IPv4 CIDR block" }, +); + +export const AwsEc2ConfigSchema = z.object({ + profile: NonEmptyStringSchema, + region: NonEmptyStringSchema, + controlCidr: AwsControlCidrSchema, + agentInstanceType: NonEmptyStringSchema, + firewallInstanceType: NonEmptyStringSchema, + agentRootVolumeGiB: z.number().int().positive(), + firewallRootVolumeGiB: z.number().int().positive(), + nixosAmiOwnerId: NonEmptyStringSchema, + nixosAmiNamePattern: NonEmptyStringSchema, +}).strict(); + +export type AwsEc2Config = Readonly>; + +function isIpv4Cidr(value: string): boolean { + const parts = value.split("/"); + if (parts.length !== 2) { + return false; + } + const [address, prefix] = parts; + if (address === undefined || prefix === undefined || !/^[0-9]+$/.test(prefix)) { + return false; + } + const prefixLength = Number(prefix); + if (!Number.isInteger(prefixLength) || prefixLength < 0 || prefixLength > 32) { + return false; + } + const octets = address.split("."); + if (octets.length !== 4) { + return false; + } + return octets.every((octet) => /^[0-9]+$/.test(octet) && Number(octet) <= 255); +} + export const CommandResultSchema = z.object({ status: NonNegativeSafeIntegerSchema, stdout: z.string(), @@ -40,6 +83,8 @@ export const RootcellConfigSchema = z.object({ imageManifestUrl: NonEmptyStringSchema, imageDir: NonEmptyStringSchema.optional(), awsSecretsManagerProviders: z.array(AwsSecretsManagerSecretProviderConfigSchema), + vmProvider: RootcellVmProviderIdSchema, + awsEc2: AwsEc2ConfigSchema.optional(), }); export type RootcellConfig = Readonly>; diff --git a/vitest.config.ts b/vitest.config.ts index f7a3d01..7e8ca30 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -20,8 +20,8 @@ export default defineConfig({ include: ["src/rootcell/integration/**/*.integration.test.ts"], fileParallelism: false, isolate: false, - testTimeout: 250_000, - hookTimeout: 250_000, + testTimeout: 1_800_000, + hookTimeout: 1_800_000, }, }, ],