From f16155aae56d24c1e7b8a6722f5b4938f19aa6d7 Mon Sep 17 00:00:00 2001
From: mulatta <67085791+mulatta@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:04:20 +0900
Subject: [PATCH 1/6] terraform: stop publishing internal service names

Keep Cloudflare DNS limited to endpoints that need public resolution so private services cannot be discovered through the public zone.
---
 terraform/cloudflare/main.tf | 40 ------------------------------------
 1 file changed, 40 deletions(-)

diff --git a/terraform/cloudflare/main.tf b/terraform/cloudflare/main.tf
index 45930ef..e7fe720 100644
--- a/terraform/cloudflare/main.tf
+++ b/terraform/cloudflare/main.tf
@@ -54,16 +54,6 @@ resource "cloudflare_dns_record" "buildbot" {
   comment = "Nixbot CI/CD edge proxy (eta -> psi)"
 }
 
-resource "cloudflare_dns_record" "s3" {
-  zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"]
-  name    = "s3.sjanglab.org"
-  content = "141.164.53.203"
-  type    = "A"
-  ttl     = 300
-  proxied = false
-  comment = "MinIO S3 API"
-}
-
 resource "cloudflare_dns_record" "ntfy" {
   zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"]
   name    = "ntfy.sjanglab.org"
@@ -74,16 +64,6 @@ resource "cloudflare_dns_record" "ntfy" {
   comment = "ntfy notification service"
 }
 
-resource "cloudflare_dns_record" "logging" {
-  zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"]
-  name    = "logging.sjanglab.org"
-  content = "141.164.53.203"
-  type    = "A"
-  ttl     = 300
-  proxied = false
-  comment = "Grafana logging dashboard"
-}
-
 resource "cloudflare_dns_record" "headscale" {
   zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"]
   name    = "hs.sjanglab.org"
@@ -104,26 +84,6 @@ resource "cloudflare_dns_record" "authentik" {
   comment = "Authentik SSO server"
 }
 
-resource "cloudflare_dns_record" "vaultwarden" {
-  zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"]
-  name    = "vault.sjanglab.org"
-  content = "141.164.53.203"
-  type    = "A"
-  ttl     = 300
-  proxied = false
-  comment = "Vaultwarden password manager"
-}
-
-resource "cloudflare_dns_record" "gatus" {
-  zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"]
-  name    = "gatus.sjanglab.org"
-  content = "141.164.53.203"
-  type    = "A"
-  ttl     = 300
-  proxied = false
-  comment = "Gatus status monitoring"
-}
-
 resource "cloudflare_dns_record" "n8n" {
   zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"]
   name    = "n8n.sjanglab.org"

From fc1df5de7cb00762dd6f6aea32d30c9f03550857 Mon Sep 17 00:00:00 2001
From: mulatta <67085791+mulatta@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:05:48 +0900
Subject: [PATCH 2/6] monitoring: serve Grafana through private auth

Keep dashboards reachable to administrators without relying on a public Cloudflare record or eta public ingress.
---
 hosts/eta.nix                        | 14 ++++-
 hosts/rho.nix                        |  1 +
 modules/headscale/acl-rules.nix      |  1 +
 modules/headscale/default.nix        |  5 ++
 modules/monitoring/reverse-proxy.nix | 77 ++++++++++++++++------------
 5 files changed, 64 insertions(+), 34 deletions(-)

diff --git a/hosts/eta.nix b/hosts/eta.nix
index 82afe80..2030a98 100644
--- a/hosts/eta.nix
+++ b/hosts/eta.nix
@@ -14,7 +14,6 @@ in
     ../modules/uptermd
     ../modules/gatus
     ../modules/monitoring/vector
-    ../modules/monitoring/reverse-proxy.nix
     ../modules/n8n/reverse-proxy.nix
     ../modules/acme/sync.nix
   ];
@@ -36,6 +35,12 @@ in
       serviceName = "acme-sync-docling-to-psi";
       remoteHost = hosts.psi.wg-admin;
     }
+    {
+      domain = "logging.sjanglab.org";
+      serviceName = "acme-sync-logging-to-rho";
+      remoteUser = "acme-sync-logging";
+      remoteHost = hosts.rho.wg-admin;
+    }
     {
       domain = "vllm.sjanglab.org";
       serviceName = "acme-sync-vllm-to-psi";
@@ -61,6 +66,13 @@ in
     group = "acme";
   };
 
+  security.acme.certs."logging.sjanglab.org" = {
+    dnsProvider = "cloudflare";
+    environmentFile = config.sops.secrets.cloudflare-credentials.path;
+    webroot = null;
+    group = "acme";
+  };
+
   networking.hostName = "eta";
   system.stateVersion = "25.05";
 }
diff --git a/hosts/rho.nix b/hosts/rho.nix
index 967598e..ba830e1 100644
--- a/hosts/rho.nix
+++ b/hosts/rho.nix
@@ -9,6 +9,7 @@
     ../modules/borgbackup/rho/client.nix
     ../modules/borgbackup/mirror.nix
     ../modules/monitoring/vector/monitor-systems.nix
+    ../modules/monitoring/reverse-proxy.nix
   ];
 
   disko.rootDisk = "/dev/disk/by-id/nvme-eui.00000000000000006479a79cdac0038a";
diff --git a/modules/headscale/acl-rules.nix b/modules/headscale/acl-rules.nix
index 284cb98..f1fb573 100644
--- a/modules/headscale/acl-rules.nix
+++ b/modules/headscale/acl-rules.nix
@@ -26,6 +26,7 @@
         "tag:ai:443"
         "tag:apps:80"
         "tag:apps:443"
+        "tag:monitoring:443"
         "tag:monitoring:3000"
       ];
     }
diff --git a/modules/headscale/default.nix b/modules/headscale/default.nix
index e210b8e..f30eb0f 100644
--- a/modules/headscale/default.nix
+++ b/modules/headscale/default.nix
@@ -56,6 +56,11 @@ in
             type = "A";
             value = "100.64.0.1"; # psi headscale IP
           }
+          {
+            name = "logging.sjanglab.org";
+            type = "A";
+            value = "100.64.0.2"; # rho headscale IP
+          }
           {
             name = "upterm.sjanglab.org";
             type = "A";
diff --git a/modules/monitoring/reverse-proxy.nix b/modules/monitoring/reverse-proxy.nix
index d22354d..769d824 100644
--- a/modules/monitoring/reverse-proxy.nix
+++ b/modules/monitoring/reverse-proxy.nix
@@ -1,44 +1,55 @@
-# Grafana reverse proxy (deployed on eta)
-# Proxies requests to rho where Grafana is running on wg-admin
-{ config, ... }:
+# Grafana tailnet reverse proxy (deployed on rho)
+{
+  config,
+  ...
+}:
 let
-  inherit (config.networking.sbee) hosts;
+  inherit (config.networking.sbee) currentHost hosts;
+  authentikAuth = import ../authentik/nginx-locations.nix { inherit hosts; };
   loggingDomain = "logging.sjanglab.org";
+  certDir = "/var/lib/acme/${loggingDomain}";
 in
 {
-  imports = [ ../acme ];
+  imports = [ ../acme/sync.nix ];
 
-  services.nginx.virtualHosts.${loggingDomain} = {
-    forceSSL = true;
-    useACMEHost = loggingDomain;
+  acmeSyncer.mkReceiver = [
+    {
+      domain = loggingDomain;
+      user = "acme-sync-logging";
+    }
+  ];
 
-    locations = {
-      "/" = {
-        proxyPass = "http://${hosts.rho.wg-admin}:3000";
-        extraConfig = ''
-          proxy_set_header Host $host;
-          proxy_set_header X-Real-IP $remote_addr;
-          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          proxy_set_header X-Forwarded-Proto $scheme;
-        '';
-      };
-      # Grafana live (WebSocket)
-      "/api/live/" = {
-        proxyPass = "http://${hosts.rho.wg-admin}:3000";
-        proxyWebsockets = true;
-        extraConfig = ''
-          proxy_set_header Host $host;
-          proxy_set_header X-Real-IP $remote_addr;
-          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-          proxy_set_header X-Forwarded-Proto $scheme;
-        '';
+  services.nginx = {
+    enable = true;
+
+    virtualHosts.${loggingDomain} = {
+      forceSSL = true;
+      sslCertificate = "${certDir}/fullchain.pem";
+      sslCertificateKey = "${certDir}/key.pem";
+
+      locations = authentikAuth.locations // {
+        "/" = {
+          proxyPass = "http://${currentHost.wg-admin}:3000";
+          extraConfig = authentikAuth.protectLocation + ''
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+          '';
+        };
+        "/api/live/" = {
+          proxyPass = "http://${currentHost.wg-admin}:3000";
+          proxyWebsockets = true;
+          extraConfig = authentikAuth.protectLocation + ''
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+          '';
+        };
       };
     };
   };
 
-  security.acme.certs.${loggingDomain} = {
-    dnsProvider = "cloudflare";
-    environmentFile = config.sops.secrets.cloudflare-credentials.path;
-    group = "nginx";
-  };
+  networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ];
 }

From 056b992da5684b34770d5ac422a3d3f7fe0d1319 Mon Sep 17 00:00:00 2001
From: mulatta <67085791+mulatta@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:06:12 +0900
Subject: [PATCH 3/6] gatus: route dashboard through private auth

Keep health pushes on the private WireGuard path and expose the dashboard only through a Headscale-resolved Authentik-protected vhost.
---
 hosts/eta.nix                   | 13 ++++++++++
 hosts/rho.nix                   |  1 +
 modules/gatus/check.nix         |  5 +---
 modules/gatus/default.nix       | 42 +++++---------------------------
 modules/gatus/reverse-proxy.nix | 43 +++++++++++++++++++++++++++++++++
 modules/headscale/default.nix   |  5 ++++
 6 files changed, 69 insertions(+), 40 deletions(-)
 create mode 100644 modules/gatus/reverse-proxy.nix

diff --git a/hosts/eta.nix b/hosts/eta.nix
index 2030a98..1393bd7 100644
--- a/hosts/eta.nix
+++ b/hosts/eta.nix
@@ -35,6 +35,12 @@ in
       serviceName = "acme-sync-docling-to-psi";
       remoteHost = hosts.psi.wg-admin;
     }
+    {
+      domain = "gatus.sjanglab.org";
+      serviceName = "acme-sync-gatus-to-rho";
+      remoteUser = "acme-sync-gatus";
+      remoteHost = hosts.rho.wg-admin;
+    }
     {
       domain = "logging.sjanglab.org";
       serviceName = "acme-sync-logging-to-rho";
@@ -66,6 +72,13 @@ in
     group = "acme";
   };
 
+  security.acme.certs."gatus.sjanglab.org" = {
+    dnsProvider = "cloudflare";
+    environmentFile = config.sops.secrets.cloudflare-credentials.path;
+    webroot = null;
+    group = "acme";
+  };
+
   security.acme.certs."logging.sjanglab.org" = {
     dnsProvider = "cloudflare";
     environmentFile = config.sops.secrets.cloudflare-credentials.path;
diff --git a/hosts/rho.nix b/hosts/rho.nix
index ba830e1..4d4fa2c 100644
--- a/hosts/rho.nix
+++ b/hosts/rho.nix
@@ -10,6 +10,7 @@
     ../modules/borgbackup/mirror.nix
     ../modules/monitoring/vector/monitor-systems.nix
     ../modules/monitoring/reverse-proxy.nix
+    ../modules/gatus/reverse-proxy.nix
   ];
 
   disko.rootDisk = "/dev/disk/by-id/nvme-eui.00000000000000006479a79cdac0038a";
diff --git a/modules/gatus/check.nix b/modules/gatus/check.nix
index 343ebdf..574583f 100644
--- a/modules/gatus/check.nix
+++ b/modules/gatus/check.nix
@@ -18,7 +18,7 @@
 }:
 let
   cfg = config.gatusCheck;
-  gatusApi = "https://gatus.sjanglab.org";
+  gatusApi = "http://${config.networking.sbee.hosts.eta.wg-admin}:8081";
 
   # Gatus external endpoint key: "${group}_${name}" with spaces/special chars → hyphens
   mkKey =
@@ -135,9 +135,6 @@ in
       message = "gatusCheck.push '${ep.name}': exactly one of 'url' or 'systemdService' must be set";
     }) cfg.push;
 
-    # Resolve gatus.sjanglab.org → eta WG IP for hosts behind NAT/WG
-    networking.hosts.${config.networking.sbee.hosts.eta.wg-admin} = [ "gatus.sjanglab.org" ];
-
     sops.secrets.gatus-push-token = {
       sopsFile = ./secrets.yaml;
     };
diff --git a/modules/gatus/default.nix b/modules/gatus/default.nix
index b15270d..634c5dd 100644
--- a/modules/gatus/default.nix
+++ b/modules/gatus/default.nix
@@ -1,22 +1,21 @@
 { config, lib, ... }:
 let
   inherit (config.networking.sbee) hosts;
-  domain = "gatus.sjanglab.org";
   port = 8081; # 8080 is used by headscale
   systemCollector = hosts.rho.wg-admin;
   cfg = config.gatusCheck;
 in
 {
-  imports = [
-    ../acme
-    ./check.nix
-  ];
+  imports = [ ./check.nix ];
 
   services.gatus = {
     enable = true;
     environmentFile = config.sops.secrets.gatus-env.path;
     settings = {
-      web.port = port;
+      web = {
+        address = "0.0.0.0";
+        inherit port;
+      };
       metrics = true;
 
       security.basic = {
@@ -63,9 +62,7 @@ in
           # psi
           (mkExtEndpoint "Nixbot" "ci")
           (mkExtEndpoint "Nixbot PostgreSQL" "ci")
-          (mkExtEndpoint "Ollama" "ai")
           (mkExtEndpoint "Docling" "ai")
-          (mkExtEndpoint "vLLM" "ai")
           # tau
           (mkExtEndpoint "Nextcloud" "apps")
           (mkExtEndpoint "n8n" "apps")
@@ -82,30 +79,6 @@ in
     sopsFile = ./secrets.yaml;
   };
 
-  # ACME certificate (DNS challenge via Cloudflare)
-  security.acme.certs.${domain} = {
-    dnsProvider = "cloudflare";
-    environmentFile = config.sops.secrets.cloudflare-credentials.path;
-    webroot = null;
-    group = "nginx";
-  };
-
-  # Nginx reverse proxy
-  services.nginx.virtualHosts.${domain} = {
-    forceSSL = true;
-    useACMEHost = domain;
-    locations."/" = {
-      proxyPass = "http://127.0.0.1:${toString port}";
-      proxyWebsockets = true;
-      extraConfig = ''
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-      '';
-    };
-  };
-
   # Vector: scrape Gatus /metrics → push to rho Prometheus
   # Independent source+sink pair, does not interfere with existing Vector pipeline
   services.vector.settings = {
@@ -123,8 +96,5 @@ in
     };
   };
 
-  networking.firewall.allowedTCPPorts = [
-    80
-    443
-  ];
+  networking.firewall.interfaces.wg-admin.allowedTCPPorts = [ port ];
 }
diff --git a/modules/gatus/reverse-proxy.nix b/modules/gatus/reverse-proxy.nix
new file mode 100644
index 0000000..1d59f5d
--- /dev/null
+++ b/modules/gatus/reverse-proxy.nix
@@ -0,0 +1,43 @@
+# Gatus tailnet reverse proxy (deployed on rho)
+{ config, ... }:
+let
+  inherit (config.networking.sbee) hosts;
+  authentikAuth = import ../authentik/nginx-locations.nix { inherit hosts; };
+  domain = "gatus.sjanglab.org";
+  certDir = "/var/lib/acme/${domain}";
+in
+{
+  imports = [ ../acme/sync.nix ];
+
+  acmeSyncer.mkReceiver = [
+    {
+      inherit domain;
+      user = "acme-sync-gatus";
+    }
+  ];
+
+  services.nginx = {
+    enable = true;
+
+    virtualHosts.${domain} = {
+      forceSSL = true;
+      sslCertificate = "${certDir}/fullchain.pem";
+      sslCertificateKey = "${certDir}/key.pem";
+
+      locations = authentikAuth.locations // {
+        "/" = {
+          proxyPass = "http://${hosts.eta.wg-admin}:8081";
+          proxyWebsockets = true;
+          extraConfig = authentikAuth.protectLocation + ''
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+          '';
+        };
+      };
+    };
+  };
+
+  networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ];
+}
diff --git a/modules/headscale/default.nix b/modules/headscale/default.nix
index f30eb0f..14028ee 100644
--- a/modules/headscale/default.nix
+++ b/modules/headscale/default.nix
@@ -56,6 +56,11 @@ in
             type = "A";
             value = "100.64.0.1"; # psi headscale IP
           }
+          {
+            name = "gatus.sjanglab.org";
+            type = "A";
+            value = "100.64.0.2"; # rho headscale IP
+          }
           {
             name = "logging.sjanglab.org";
             type = "A";

From 30e5d74a3d06bad3f260f7c17eb11be86dbedb15 Mon Sep 17 00:00:00 2001
From: mulatta <67085791+mulatta@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:06:47 +0900
Subject: [PATCH 4/6] vaultwarden: keep password access inside the tailnet

Preserve the existing eta data service while moving browser ingress behind a Headscale-resolved tau proxy.
---
 hosts/eta.nix                         | 13 +++++++++
 hosts/tau.nix                         |  1 +
 modules/headscale/default.nix         |  5 ++++
 modules/vaultwarden/default.nix       | 39 ++++----------------------
 modules/vaultwarden/reverse-proxy.nix | 40 +++++++++++++++++++++++++++
 5 files changed, 64 insertions(+), 34 deletions(-)
 create mode 100644 modules/vaultwarden/reverse-proxy.nix

diff --git a/hosts/eta.nix b/hosts/eta.nix
index 1393bd7..969be8d 100644
--- a/hosts/eta.nix
+++ b/hosts/eta.nix
@@ -47,6 +47,12 @@ in
       remoteUser = "acme-sync-logging";
       remoteHost = hosts.rho.wg-admin;
     }
+    {
+      domain = "vault.sjanglab.org";
+      serviceName = "acme-sync-vaultwarden-to-tau";
+      remoteUser = "acme-sync-vaultwarden";
+      remoteHost = hosts.tau.wg-admin;
+    }
     {
       domain = "vllm.sjanglab.org";
       serviceName = "acme-sync-vllm-to-psi";
@@ -86,6 +92,13 @@ in
     group = "acme";
   };
 
+  security.acme.certs."vault.sjanglab.org" = {
+    dnsProvider = "cloudflare";
+    environmentFile = config.sops.secrets.cloudflare-credentials.path;
+    webroot = null;
+    group = "acme";
+  };
+
   networking.hostName = "eta";
   system.stateVersion = "25.05";
 }
diff --git a/hosts/tau.nix b/hosts/tau.nix
index eadbe88..c1bd393 100644
--- a/hosts/tau.nix
+++ b/hosts/tau.nix
@@ -11,6 +11,7 @@
     ../modules/monitoring/vector/monitor-services.nix
     ../modules/nextcloud
     ../modules/n8n
+    ../modules/vaultwarden/reverse-proxy.nix
   ];
 
   disko.rootDisk = "/dev/disk/by-id/nvme-eui.00000000000000006479a79cdac0038f";
diff --git a/modules/headscale/default.nix b/modules/headscale/default.nix
index 14028ee..7032f10 100644
--- a/modules/headscale/default.nix
+++ b/modules/headscale/default.nix
@@ -66,6 +66,11 @@ in
             type = "A";
             value = "100.64.0.2"; # rho headscale IP
           }
+          {
+            name = "vault.sjanglab.org";
+            type = "A";
+            value = "100.64.0.3"; # tau headscale IP
+          }
           {
             name = "upterm.sjanglab.org";
             type = "A";
diff --git a/modules/vaultwarden/default.nix b/modules/vaultwarden/default.nix
index 779b569..f817f19 100644
--- a/modules/vaultwarden/default.nix
+++ b/modules/vaultwarden/default.nix
@@ -1,15 +1,13 @@
 { config, ... }:
 {
-  imports = [
-    ../acme
-    ../gatus/check.nix
-  ];
+  imports = [ ../gatus/check.nix ];
 
   gatusCheck.pull = [
     {
       name = "Vaultwarden";
-      url = "https://vault.sjanglab.org/alive";
+      url = "http://127.0.0.1:8000/alive";
       group = "apps";
+      conditions = [ "[STATUS] == 200" ];
     }
   ];
 
@@ -35,6 +33,7 @@
       # Organization
       ORG_CREATION_USERS = "sjang.bioe@gmail.com,admin@sjanglab.org";
 
+      ROCKET_ADDRESS = "0.0.0.0";
       ROCKET_PORT = 8000;
     };
   };
@@ -46,37 +45,9 @@
     mode = "0400";
   };
 
-  # ACME certificate (DNS challenge)
-  security.acme.certs."vault.sjanglab.org" = {
-    dnsProvider = "cloudflare";
-    environmentFile = config.sops.secrets.cloudflare-credentials.path;
-    webroot = null;
-    group = "nginx";
-  };
-
-  # Nginx reverse proxy
-  services.nginx.virtualHosts."vault.sjanglab.org" = {
-    forceSSL = true;
-    useACMEHost = "vault.sjanglab.org";
-
-    locations."/" = {
-      proxyPass = "http://127.0.0.1:8000";
-      proxyWebsockets = true;
-      extraConfig = ''
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-      '';
-    };
-  };
-
   systemd.tmpfiles.rules = [
     "d /var/backup/vaultwarden 0700 vaultwarden vaultwarden -"
   ];
 
-  networking.firewall.allowedTCPPorts = [
-    80
-    443
-  ];
+  networking.firewall.interfaces.wg-admin.allowedTCPPorts = [ 8000 ];
 }
diff --git a/modules/vaultwarden/reverse-proxy.nix b/modules/vaultwarden/reverse-proxy.nix
new file mode 100644
index 0000000..a23b71f
--- /dev/null
+++ b/modules/vaultwarden/reverse-proxy.nix
@@ -0,0 +1,40 @@
+# Vaultwarden tailnet reverse proxy (deployed on tau)
+{ config, ... }:
+let
+  inherit (config.networking.sbee) hosts;
+  domain = "vault.sjanglab.org";
+  certDir = "/var/lib/acme/${domain}";
+in
+{
+  imports = [ ../acme/sync.nix ];
+
+  acmeSyncer.mkReceiver = [
+    {
+      inherit domain;
+      user = "acme-sync-vaultwarden";
+    }
+  ];
+
+  services.nginx = {
+    enable = true;
+
+    virtualHosts.${domain} = {
+      forceSSL = true;
+      sslCertificate = "${certDir}/fullchain.pem";
+      sslCertificateKey = "${certDir}/key.pem";
+
+      locations."/" = {
+        proxyPass = "http://${hosts.eta.wg-admin}:8000";
+        proxyWebsockets = true;
+        extraConfig = ''
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header X-Forwarded-Proto $scheme;
+        '';
+      };
+    };
+  };
+
+  networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ];
+}

From 2bdc09e505c0e9da5c2bb459dc679297e590281e Mon Sep 17 00:00:00 2001
From: mulatta <67085791+mulatta@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:07:08 +0900
Subject: [PATCH 5/6] ai: keep inactive API services dormant

Avoid renewing certificates for disabled Ollama and vLLM deployments while documenting that API clients need token-style auth instead of browser redirects.
---
 hosts/eta.nix              | 26 --------------------------
 modules/ollama/default.nix |  3 ++-
 modules/vllm/default.nix   |  2 ++
 3 files changed, 4 insertions(+), 27 deletions(-)

diff --git a/hosts/eta.nix b/hosts/eta.nix
index 969be8d..4a2f37c 100644
--- a/hosts/eta.nix
+++ b/hosts/eta.nix
@@ -24,12 +24,6 @@ in
       serviceName = "acme-sync-to-tau";
       remoteHost = hosts.tau.wg-admin;
     }
-    {
-      domain = "ollama.sjanglab.org";
-      serviceName = "acme-sync-ollama-to-psi";
-      remoteUser = "acme-sync-ollama";
-      remoteHost = hosts.psi.wg-admin;
-    }
     {
       domain = "docling.sjanglab.org";
       serviceName = "acme-sync-docling-to-psi";
@@ -53,31 +47,11 @@ in
       remoteUser = "acme-sync-vaultwarden";
       remoteHost = hosts.tau.wg-admin;
     }
-    {
-      domain = "vllm.sjanglab.org";
-      serviceName = "acme-sync-vllm-to-psi";
-      remoteUser = "acme-sync-vllm";
-      remoteHost = hosts.psi.wg-admin;
-    }
   ];
 
   disko.rootDisk = "/dev/vda";
 
   # ACME certificates for internal services
-  security.acme.certs."ollama.sjanglab.org" = {
-    dnsProvider = "cloudflare";
-    environmentFile = config.sops.secrets.cloudflare-credentials.path;
-    webroot = null;
-    group = "acme";
-  };
-
-  security.acme.certs."vllm.sjanglab.org" = {
-    dnsProvider = "cloudflare";
-    environmentFile = config.sops.secrets.cloudflare-credentials.path;
-    webroot = null;
-    group = "acme";
-  };
-
   security.acme.certs."gatus.sjanglab.org" = {
     dnsProvider = "cloudflare";
     environmentFile = config.sops.secrets.cloudflare-credentials.path;
diff --git a/modules/ollama/default.nix b/modules/ollama/default.nix
index c34965f..471ebf5 100644
--- a/modules/ollama/default.nix
+++ b/modules/ollama/default.nix
@@ -94,7 +94,8 @@ in
       sslCertificate = "${certDir}/fullchain.pem";
       sslCertificateKey = "${certDir}/key.pem";
 
-      # Access control: Headscale ACL (network-level, no forward auth)
+      # API clients do not handle browser redirects from Authentik forward auth.
+      # Keep application authorization in API credentials when this module is enabled.
       locations."/" = {
         proxyPass = "http://127.0.0.1:${toString port}";
         recommendedProxySettings = false; # Override Host header manually
diff --git a/modules/vllm/default.nix b/modules/vllm/default.nix
index fa3a265..b29e453 100644
--- a/modules/vllm/default.nix
+++ b/modules/vllm/default.nix
@@ -135,6 +135,8 @@ in
       sslCertificate = "/var/lib/acme/${domain}/fullchain.pem";
       sslCertificateKey = "/var/lib/acme/${domain}/key.pem";
 
+      # API clients do not handle browser redirects from Authentik forward auth.
+      # Keep application authorization in API credentials when this module is enabled.
       locations."/" = {
         proxyPass = "http://127.0.0.1:8100";
         extraConfig = ''

From 7ff5479209089b50f3040a67f832d107ebad340d Mon Sep 17 00:00:00 2001
From: mulatta <67085791+mulatta@users.noreply.github.com>
Date: Fri, 3 Jul 2026 00:08:35 +0900
Subject: [PATCH 6/6] multievolve: gate service through private auth

Expose the Streamlit UI only after Headscale routing and Authentik policy so research access does not depend on public ingress.
---
 flake.lock                      | 180 ++++++++++++++++++++++++++++++--
 flake.nix                       |   2 +
 hosts/eta.nix                   |  13 +++
 hosts/psi.nix                   |   1 +
 modules/headscale/default.nix   |   5 +
 modules/multievolve/default.nix |  82 +++++++++++++++
 6 files changed, 277 insertions(+), 6 deletions(-)
 create mode 100644 modules/multievolve/default.nix

diff --git a/flake.lock b/flake.lock
index 2f39763..f1d330f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -148,6 +148,27 @@
         "type": "github"
       }
     },
+    "multievolve-nix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "treefmt-nix": "treefmt-nix",
+        "uv2nix-env": "uv2nix-env"
+      },
+      "locked": {
+        "lastModified": 1782993998,
+        "narHash": "sha256-lMjTMaU3vB0hhGVHF6Q4kA+Ti5qYuUQbs6dtdq0VEgU=",
+        "owner": "mulatta",
+        "repo": "multievolve-nix",
+        "rev": "618b1f418073d4698c1eb1e4f41a2038dd8bb9ae",
+        "type": "github"
+      },
+      "original": {
+        "owner": "mulatta",
+        "ref": "nixos-module-service",
+        "repo": "multievolve-nix",
+        "type": "github"
+      }
+    },
     "niks3": {
       "inputs": {
         "nixpkgs": [
@@ -260,16 +281,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1782847225,
-        "narHash": "sha256-JC9PjqKYG9ve5U8aDOLQipp3+KLANBHUvGdLZlxzdKI=",
+        "lastModified": 1782723713,
+        "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "95ca1e203c0750115fd4a6f17d5a245dfe6b1edd",
+        "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-26.05",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -290,6 +311,22 @@
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1782847225,
+        "narHash": "sha256-JC9PjqKYG9ve5U8aDOLQipp3+KLANBHUvGdLZlxzdKI=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "95ca1e203c0750115fd4a6f17d5a245dfe6b1edd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-26.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "pyproject-build-systems": {
       "inputs": {
         "nixpkgs": [
@@ -319,6 +356,38 @@
         "type": "github"
       }
     },
+    "pyproject-build-systems_2": {
+      "inputs": {
+        "nixpkgs": [
+          "multievolve-nix",
+          "uv2nix-env",
+          "nixpkgs"
+        ],
+        "pyproject-nix": [
+          "multievolve-nix",
+          "uv2nix-env",
+          "pyproject-nix"
+        ],
+        "uv2nix": [
+          "multievolve-nix",
+          "uv2nix-env",
+          "uv2nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1782093830,
+        "narHash": "sha256-6gmEVe69+KlRkZD4PEEV5xAlB9CB0Y9TiuEgQjDrKTQ=",
+        "owner": "pyproject-nix",
+        "repo": "build-system-pkgs",
+        "rev": "430680a19bc85a3bda55f12e4cc1a1aadcf2e478",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "build-system-pkgs",
+        "type": "github"
+      }
+    },
     "pyproject-nix": {
       "inputs": {
         "nixpkgs": [
@@ -340,21 +409,44 @@
         "type": "github"
       }
     },
+    "pyproject-nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "multievolve-nix",
+          "uv2nix-env",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1782808548,
+        "narHash": "sha256-9+XcLOzFo3tXSgH5xpd5g2i65cx4q6zvfNVEMV7hyEs=",
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "rev": "4de7c126c1a3e76e9ec9e3ced900aaa6d8847757",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "authentik-nix": "authentik-nix",
         "disko": "disko",
         "fast-nix-gc": "fast-nix-gc",
         "flake-parts": "flake-parts",
+        "multievolve-nix": "multievolve-nix",
         "niks3": "niks3",
         "nix-index-database": "nix-index-database",
         "nixbot": "nixbot",
         "nixos-images": "nixos-images",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "nixpkgs-unstable": "nixpkgs-unstable",
         "sops-nix": "sops-nix",
         "srvos": "srvos",
-        "treefmt-nix": "treefmt-nix"
+        "treefmt-nix": "treefmt-nix_2"
       }
     },
     "sops-nix": {
@@ -413,6 +505,27 @@
       }
     },
     "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "multievolve-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1780220602,
+        "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "db947814a175b7ca6ded66e21383d938df01c227",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "treefmt-nix_2": {
       "inputs": {
         "nixpkgs": [
           "nixpkgs"
@@ -456,6 +569,61 @@
         "repo": "uv2nix",
         "type": "github"
       }
+    },
+    "uv2nix-env": {
+      "inputs": {
+        "nixpkgs": [
+          "multievolve-nix",
+          "nixpkgs"
+        ],
+        "pyproject-build-systems": "pyproject-build-systems_2",
+        "pyproject-nix": "pyproject-nix_2",
+        "treefmt-nix": [
+          "multievolve-nix",
+          "treefmt-nix"
+        ],
+        "uv2nix": "uv2nix_2"
+      },
+      "locked": {
+        "lastModified": 1782880978,
+        "narHash": "sha256-MewckjJGV0DV2X070b6T93FctV0D0DkDAg7aFfGXa8I=",
+        "owner": "mulatta",
+        "repo": "uv2nix-env",
+        "rev": "67a987d1ef13a7313dbbeea364ec71a443c3e8ce",
+        "type": "github"
+      },
+      "original": {
+        "owner": "mulatta",
+        "repo": "uv2nix-env",
+        "type": "github"
+      }
+    },
+    "uv2nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "multievolve-nix",
+          "uv2nix-env",
+          "nixpkgs"
+        ],
+        "pyproject-nix": [
+          "multievolve-nix",
+          "uv2nix-env",
+          "pyproject-nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1782810559,
+        "narHash": "sha256-loy132LKn8QfiPbFJhdTjm+yZG3zklyOQhVabLyx2l4=",
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "rev": "2f698e4b5a3c6004edaf051543542f18a36afe77",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",
diff --git a/flake.nix b/flake.nix
index 8a5ec6a..4901397 100644
--- a/flake.nix
+++ b/flake.nix
@@ -62,6 +62,8 @@
     };
 
     # Applications.
+    multievolve-nix.url = "github:mulatta/multievolve-nix/nixos-module-service";
+
     niks3 = {
       url = "github:Mic92/niks3";
       inputs.nixpkgs.follows = "nixpkgs";
diff --git a/hosts/eta.nix b/hosts/eta.nix
index 4a2f37c..1dd88f6 100644
--- a/hosts/eta.nix
+++ b/hosts/eta.nix
@@ -41,6 +41,12 @@ in
       remoteUser = "acme-sync-logging";
       remoteHost = hosts.rho.wg-admin;
     }
+    {
+      domain = "multievolve.sjanglab.org";
+      serviceName = "acme-sync-multievolve-to-psi";
+      remoteUser = "acme-sync-multievolve";
+      remoteHost = hosts.psi.wg-admin;
+    }
     {
       domain = "vault.sjanglab.org";
       serviceName = "acme-sync-vaultwarden-to-tau";
@@ -66,6 +72,13 @@ in
     group = "acme";
   };
 
+  security.acme.certs."multievolve.sjanglab.org" = {
+    dnsProvider = "cloudflare";
+    environmentFile = config.sops.secrets.cloudflare-credentials.path;
+    webroot = null;
+    group = "acme";
+  };
+
   security.acme.certs."vault.sjanglab.org" = {
     dnsProvider = "cloudflare";
     environmentFile = config.sops.secrets.cloudflare-credentials.path;
diff --git a/hosts/psi.nix b/hosts/psi.nix
index b13e5bf..9aa20bd 100644
--- a/hosts/psi.nix
+++ b/hosts/psi.nix
@@ -15,6 +15,7 @@
     ../modules/borgbackup/psi/client.nix
     ../modules/monitoring/vector
     ../modules/harmonia
+    ../modules/multievolve
     # ../modules/vllm
     ../modules/db-sync/databases.nix
     ../modules/docling
diff --git a/modules/headscale/default.nix b/modules/headscale/default.nix
index 7032f10..63b2896 100644
--- a/modules/headscale/default.nix
+++ b/modules/headscale/default.nix
@@ -66,6 +66,11 @@ in
             type = "A";
             value = "100.64.0.2"; # rho headscale IP
           }
+          {
+            name = "multievolve.sjanglab.org";
+            type = "A";
+            value = "100.64.0.1"; # psi headscale IP
+          }
           {
             name = "vault.sjanglab.org";
             type = "A";
diff --git a/modules/multievolve/default.nix b/modules/multievolve/default.nix
new file mode 100644
index 0000000..b076996
--- /dev/null
+++ b/modules/multievolve/default.nix
@@ -0,0 +1,82 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  inherit (config.networking.sbee) hosts;
+  authentikAuth = import ../authentik/nginx-locations.nix { inherit hosts; };
+  domain = "multievolve.sjanglab.org";
+  port = 8501;
+  certDir = "/var/lib/acme/${domain}";
+in
+{
+  imports = [
+    inputs.multievolve-nix.nixosModules.default
+    ../acme/sync.nix
+    ../gatus/check.nix
+  ];
+
+  gatusCheck.push = [
+    {
+      name = "MULTI-evolve";
+      group = "ai";
+      url = "http://127.0.0.1:${toString port}/_stcore/health";
+    }
+  ];
+
+  acmeSyncer.mkReceiver = [
+    {
+      inherit domain;
+      user = "acme-sync-multievolve";
+    }
+  ];
+
+  services.multievolve-streamlit = {
+    enable = true;
+    host = "127.0.0.1";
+    inherit port;
+    workingDirectory = "/workspace/multievolve";
+    extraGroups = [
+      "render"
+      "video"
+    ];
+    environment = {
+      CUDA_VISIBLE_DEVICES = "0";
+      LD_LIBRARY_PATH = lib.concatStringsSep ":" [
+        "/run/opengl-driver/lib"
+        "${pkgs.cudaPackages.cuda_cudart}/lib"
+        "${pkgs.cudaPackages.libcublas}/lib"
+      ];
+    };
+  };
+
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+    recommendedTlsSettings = true;
+
+    virtualHosts.${domain} = {
+      forceSSL = true;
+      sslCertificate = "${certDir}/fullchain.pem";
+      sslCertificateKey = "${certDir}/key.pem";
+
+      locations = authentikAuth.locations // {
+        "/" = {
+          proxyPass = "http://127.0.0.1:${toString port}";
+          proxyWebsockets = true;
+          extraConfig = authentikAuth.protectLocation + ''
+            proxy_buffering off;
+            proxy_read_timeout 600s;
+            proxy_send_timeout 600s;
+            client_max_body_size 1G;
+          '';
+        };
+      };
+    };
+  };
+
+  networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ];
+}