diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9a37339..eafed44 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - host: [finney, kingfisher] + host: [finney, kingfisher, albatross] steps: - uses: actions/checkout@v4 - uses: cachix/install-nix-action@v30 diff --git a/flake.lock b/flake.lock index e1f0fcb..da87afe 100644 --- a/flake.lock +++ b/flake.lock @@ -214,11 +214,11 @@ ] }, "locked": { - "lastModified": 1779107941, - "narHash": "sha256-uqcr+MR6f+EAxmWedrezCOirUTMEjaT47sl+8Wg1Glk=", + "lastModified": 1779190854, + "narHash": "sha256-c2G3VCBrqLyoTg8fzzN81vfBhW1dJHljEGCXrmohaC8=", "owner": "2140-dev", "repo": "roost", - "rev": "5a675cb898286f3cb93cc44269471c965da7e6c2", + "rev": "b9c80bebae089e0658baa5a17f65fd38f75badd7", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 4584b53..6fe4f27 100644 --- a/flake.nix +++ b/flake.nix @@ -57,8 +57,14 @@ { nixosConfigurations = { finney = mkHost "finney" [ ]; - kingfisher = mkHost "kingfisher" [ roost.nixosModules.default ]; - albatross = mkHost "albatross" [ roost.nixosModules.default ]; + kingfisher = mkHost "kingfisher" [ + roost.nixosModules.default + roost.nixosModules.wireguard-mesh + ]; + albatross = mkHost "albatross" [ + roost.nixosModules.frigate-edge + roost.nixosModules.wireguard-mesh + ]; }; formatter = forAllSystems (system: nixpkgs.legacyPackages.${system}.nixfmt-tree); diff --git a/hosts/_mesh.nix b/hosts/_mesh.nix new file mode 100644 index 0000000..bf618db --- /dev/null +++ b/hosts/_mesh.nix @@ -0,0 +1,28 @@ +{ ... }: + +# Shared peer definitions for the private WireGuard mesh between +# kingfisher (the all-in-one frigate node) and albatross (the +# frigate-edge consumer). Imported by both hosts' wireguard.nix so the +# `peers` block is one place. Adding a third node is a one-line edit +# here plus that node's own `thisHost` + private-key wiring. +# +# Endpoint IPs are kingfisher's and albatross's public addresses. WG +# resolves these once at interface setup; if either box's IP rotates +# the matching entry below has to update too. The private keys +# matching the public keys below live in secrets/wireguard-.age, +# encrypted to both `josie` and the host's own SSH key. + +{ + services.roost.wireguard-mesh.peers = { + kingfisher = { + publicKey = "65eBW/IfinjLj7Q9HBnw+CBeEAx/6zaMVDejs+Vxb2o="; + endpoint = "136.243.9.246:51820"; + meshIp = "10.42.0.1"; + }; + albatross = { + publicKey = "BZFpBTwYt3RUPFkIMQIrXZgkMDGryaae/empkoEiehE="; + endpoint = "46.62.185.45:51820"; + meshIp = "10.42.0.2"; + }; + }; +} diff --git a/hosts/albatross/default.nix b/hosts/albatross/default.nix index d39664e..3d12672 100644 --- a/hosts/albatross/default.nix +++ b/hosts/albatross/default.nix @@ -11,6 +11,7 @@ ./hardware-configuration.nix ./gpu.nix ./frigate.nix + ./wireguard.nix ]; networking.hostName = "albatross"; diff --git a/hosts/albatross/frigate.nix b/hosts/albatross/frigate.nix index afe4eac..e22589c 100644 --- a/hosts/albatross/frigate.nix +++ b/hosts/albatross/frigate.nix @@ -6,61 +6,40 @@ }: { - # Own hostname for now (rather than sharing kingfisher's - # frigate.2140.dev). Each backend self-issues its own cert via - # HTTP-01 — no DNS-01 wiring and no shared-cert coordination. - # Future load-balancing work will move both back behind a single - # hostname, fronted by a third host doing TCP/SNI passthrough. - services.public-frigate = { + # Edge-mode Frigate: TLS termination + ACME live here, the bitcoind / + # fulcrum / ZMQ stack runs on kingfisher and is consumed over the + # private WireGuard mesh (see ./wireguard.nix and ../_mesh.nix). + # This sidesteps the storage-capacity problem on albatross's + # ~866 GB pool — frigate's own DuckDB index is the only local data. + services.frigate-edge = { enable = true; host = "albatross.2140.dev"; tls.acmeEmail = "josie@2140.dev"; - }; - # Add josie to the `bitcoin` group so `bitcoin-cli` works directly - # without `sudo -u bitcoin`. Operator name varies per box, so this - # stays per-host. - nix-bitcoin.operator = { - enable = true; - name = "josie"; + backend = { + bitcoind = { + rpcUrl = "http://10.42.0.1:8332"; + authCredentialFile = config.age.secrets.bitcoind-rpc-creds.path; + zmqSequenceEndpoint = "tcp://10.42.0.1:28336"; + }; + electrumUrl = "tcp://10.42.0.1:60001"; + }; }; - # 48-thread Xeon Gold 5412U with 256 GB RAM and a Blackwell RTX PRO - # 6000 (96 GB VRAM). The roost preset's default dbCache of 4 GB - # underuses this box badly during IBD; bump it for faster initial - # sync. Drop back after sync if memory pressure shows up elsewhere. - services.bitcoind.dbCache = lib.mkForce 16384; + # `user:password` consumed by frigate via systemd LoadCredential. The + # matching `rpcauth=user:salt$hash` line lives on kingfisher in its + # `exposeBackends.rpcAuth.passwordHMAC` setting. Mode 0440 + owner + # `frigate` so the frigate user (declared by the bare frigate module) + # can read it for LoadCredential to pick up. + age.secrets.bitcoind-rpc-creds = { + file = ../../secrets/bitcoind-rpc-creds.age; + owner = "frigate"; + mode = "0440"; + }; # systemd starts services with a stripped environment that does not # inherit NixOS's interactive-shell GPU library path. Without this, # frigate's JVM dlopen of libOpenCL.so.1 fails and DuckDB's ufsecp # extension silently falls back to CPU. systemd.services.frigate.environment.LD_LIBRARY_PATH = "/run/opengl-driver/lib"; - - # batchSize tuning is deferred: this GPU is roughly an order of - # magnitude more capable than kingfisher's RTX 4000 SFF Ada and the - # right value will only be visible after a real scan. The module - # default (300_000) is the starting point. - - # BOOTSTRAP: keep the data-bearing services from autostarting on the - # first boot so /var/lib/{bitcoind,fulcrum,frigate} can be populated - # via `zfs recv` from kingfisher without racing live writes. The - # users/groups still get created (those come from the modules' user - # definitions, independent of wantedBy), so the post-recv chown - # step has somebody to chown to. - # - # Also disable autoUpgrade for the duration: the seeding workflow - # destroys the placeholder datasets before recv, which would leave - # mount units in a failed state until recv lands. An hourly rebuild - # firing during that window risks compounding the breakage and - # killed albatross on the first attempt. - # - # Remove this block once the import is complete and push; the next - # nixos-rebuild will land services in multi-user.target normally, - # they will start with the imported state, and autoUpgrade resumes - # polling. - systemd.services.bitcoind.wantedBy = lib.mkForce [ ]; - systemd.services.fulcrum.wantedBy = lib.mkForce [ ]; - systemd.services.frigate.wantedBy = lib.mkForce [ ]; - system.autoUpgrade.enable = lib.mkForce false; } diff --git a/hosts/albatross/hardware-configuration.nix b/hosts/albatross/hardware-configuration.nix index c0d2761..a2e4054 100644 --- a/hosts/albatross/hardware-configuration.nix +++ b/hosts/albatross/hardware-configuration.nix @@ -1,14 +1,25 @@ # Do not modify this file! It was generated by ‘nixos-generate-config’ # and may be overwritten by future invocations. Please make changes # to /etc/nixos/configuration.nix instead. -{ config, lib, pkgs, modulesPath, ... }: +{ + config, + lib, + pkgs, + modulesPath, + ... +}: { - imports = - [ (modulesPath + "/installer/scan/not-detected.nix") - ]; + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ]; + boot.initrd.availableKernelModules = [ + "xhci_pci" + "ahci" + "nvme" + "usbhid" + ]; boot.initrd.kernelModules = [ ]; boot.kernelModules = [ "kvm-intel" ]; boot.extraModulePackages = [ ]; diff --git a/hosts/albatross/wireguard.nix b/hosts/albatross/wireguard.nix new file mode 100644 index 0000000..2296d09 --- /dev/null +++ b/hosts/albatross/wireguard.nix @@ -0,0 +1,20 @@ +{ config, ... }: + +{ + imports = [ ../_mesh.nix ]; + + # Encrypted private key. Agenix decrypts to /run/agenix/wireguard-albatross + # at activation; the wireguard module reads it as `privateKeyFile`. Mode + # 0400 keeps it root-only on disk. + age.secrets.wireguard-albatross = { + file = ../../secrets/wireguard-albatross.age; + mode = "0400"; + }; + + services.roost.wireguard-mesh = { + enable = true; + thisHost = "albatross"; + privateKeyFile = config.age.secrets.wireguard-albatross.path; + meshCidr = "10.42.0.0/24"; + }; +} diff --git a/hosts/kingfisher/default.nix b/hosts/kingfisher/default.nix index be2a376..f23b299 100644 --- a/hosts/kingfisher/default.nix +++ b/hosts/kingfisher/default.nix @@ -11,6 +11,7 @@ ./hardware-configuration.nix ./gpu.nix ./frigate.nix + ./wireguard.nix ]; networking.hostName = "kingfisher"; diff --git a/hosts/kingfisher/frigate.nix b/hosts/kingfisher/frigate.nix index bf8e98e..9c507d8 100644 --- a/hosts/kingfisher/frigate.nix +++ b/hosts/kingfisher/frigate.nix @@ -14,6 +14,26 @@ enable = true; host = "frigate.2140.dev"; tls.acmeEmail = "josie@2140.dev"; + + # Expose bitcoind RPC + ZMQ + fulcrum on the WireGuard mesh interface + # so albatross can run frigate-edge against this stack instead of + # carrying its own ~950 GB chain copy. Interface-scoped firewall + # keeps these ports unreachable from the public internet. + # + # The HMAC below is committed (one-way derived from the password); + # the plaintext lives in secrets/bitcoind-rpc-creds.age on albatross. + # See modules/wireguard-mesh.nix for the mesh topology and + # hosts/_mesh.nix for the peer registry. + exposeBackends = { + enable = true; + bindAddress = "10.42.0.1"; + interface = "wg0"; + allowedPeers = [ "10.42.0.2/32" ]; + rpcAuth = { + user = "frigate-edge"; + passwordHMAC = "bec2842f5d4d3451316cc22f5db6560c$804448c1fd845e4160f5e6cc182b8250d5324679b9372e817fdb37c42ea71cc9"; + }; + }; }; # Operator pattern: add josie to the `bitcoin` group so `bitcoin-cli` diff --git a/hosts/kingfisher/wireguard.nix b/hosts/kingfisher/wireguard.nix new file mode 100644 index 0000000..acdb069 --- /dev/null +++ b/hosts/kingfisher/wireguard.nix @@ -0,0 +1,20 @@ +{ config, ... }: + +{ + imports = [ ../_mesh.nix ]; + + # Encrypted private key. Agenix decrypts to /run/agenix/wireguard-kingfisher + # at activation; the wireguard module reads it as `privateKeyFile`. Mode + # 0400 keeps it root-only on disk. + age.secrets.wireguard-kingfisher = { + file = ../../secrets/wireguard-kingfisher.age; + mode = "0400"; + }; + + services.roost.wireguard-mesh = { + enable = true; + thisHost = "kingfisher"; + privateKeyFile = config.age.secrets.wireguard-kingfisher.path; + meshCidr = "10.42.0.0/24"; + }; +} diff --git a/secrets/bitcoind-rpc-creds.age b/secrets/bitcoind-rpc-creds.age new file mode 100644 index 0000000..e4130ba --- /dev/null +++ b/secrets/bitcoind-rpc-creds.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> X25519 HmHFvOhvEcpO8m1QxQOapgtRxbzYBy/Ep/wtZ2YFsRs +eku0xiNGs2F6FYpwsjKJCJ9u66c5opg22mZAl/aWKoM +-> ssh-ed25519 gk326w I6Yz6OCVP/J4xYDKIaKKPrUITRpwnmfbuYj3mcgDkzI +pr67JMb6f7zIkNr8OeK1FVn3Uq3J/IanazqIzjN4PQA +--- DgrxI4XH1OG1OUFvxHSK8jPfBjs7dvXDxfRrbDNA57g +[t'>.F +5n9<!UZ5r<͸DL׊5-×/ +A!Uݽ1ziXHhvY \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index f471767..068c5d2 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -11,15 +11,32 @@ # nix run github:ryantm/agenix -- --identity edit-key # and paste its public form into `josie` below. # 4. Encrypt secrets: -# cd secrets && nix run github:ryantm/agenix -- -e bitcoind-rpcauth.age +# cd secrets && nix run github:ryantm/agenix -- -e .age # 5. Reference them from a host module via `age.secrets..file = ./secrets/.age;`. let - josie = "age1...REPLACE_ME_WITH_YOUR_AGE_PUBKEY"; + josie = "age1jf8np2gw2wkd0k46x4z3plr47jz0kqvjker63jh2xqqjqpszcedsg2e6ug"; finney = "ssh-ed25519 AAAA...REPLACE_ME_WITH_HOST_KEY_AFTER_FIRST_BOOT"; - kingfisher = "ssh-ed25519 AAAA...REPLACE_ME_WITH_HOST_KEY_AFTER_FIRST_BOOT"; + kingfisher = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB2j+A4rvxr+5JIP4XrRqAI3uHUOriAPpiDSc8F+izAG root@kingfisher"; + albatross = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAPnn6DYBcz7nkpnOniTfwLtncQ8JlzYSjkFLd5uL5o3 root@albatross"; in { - # Examples — uncomment and create the .age files when wiring services: - # "bitcoind-rpcauth.age".publicKeys = [ josie kingfisher ]; - # "wireguard-finney.age".publicKeys = [ josie finney ]; + # `user:password` for the bitcoind RPC user that frigate-edge on + # albatross uses to authenticate to kingfisher's bitcoind. The + # corresponding rpcauth HMAC lives in + # hosts/kingfisher/frigate.nix (services.public-frigate.exposeBackends.rpcAuth.passwordHMAC). + "bitcoind-rpc-creds.age".publicKeys = [ + josie + albatross + ]; + + # Per-host WireGuard private keys. Each one only needs to decrypt on + # its own host plus josie (so josie can re-encrypt if needed). + "wireguard-kingfisher.age".publicKeys = [ + josie + kingfisher + ]; + "wireguard-albatross.age".publicKeys = [ + josie + albatross + ]; } diff --git a/secrets/wireguard-albatross.age b/secrets/wireguard-albatross.age new file mode 100644 index 0000000..b94ba18 --- /dev/null +++ b/secrets/wireguard-albatross.age @@ -0,0 +1,8 @@ +age-encryption.org/v1 +-> X25519 vuYGyBlUd9JWV2NmLJh1/y15DPqQCF0jx41c/6Rq/Rk +ys9jOAiTgWsZ6JLqxJgaPr8s3uTfbws1P04W6VAWfH0 +-> ssh-ed25519 gk326w PeBwSV+7Ip6wTChUOSBQvq8/4vTaK47s9RnbT4FtEUc +Rz0l+0w68Pn/xkV3YjktyBjORrhwWwolxLlTgSZjuwI +--- 2PZ/h/XsrH2acXB4THqEZW+0qYNQcY90k75l5L1RwTY +xCф +o,| `_%3Op>Ւ=#WZ_d#fvkŏcoĉ/a \ No newline at end of file diff --git a/secrets/wireguard-kingfisher.age b/secrets/wireguard-kingfisher.age new file mode 100644 index 0000000..4cffa16 --- /dev/null +++ b/secrets/wireguard-kingfisher.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> X25519 D1Ve7RKEJVkOsoblZrgyTd6k9c1fpQQrUnMkfmlYSyM +/8/jYX7P0Iw/Js4/TqivBXz1q4g3XhR5uHpTrFDjRy0 +-> ssh-ed25519 8eNJZw 3Bo+7+XQgVpwZSY1KvT9P0GOgUHL6YmfakBHUdD1vQc +GM3Od97IzBefnZ99Fc+iUWtvjcoE5baK55FEYRsRfsc +--- XPebRbSZsB1LpCm27WstMLbWgwpzsgitiekJN/8gcPE +yÏ]uyl~55u}-S&72T^ c|k[e'>,|>c@0$Ԭ` \ No newline at end of file