diff --git a/docs/admin/monitoring.md b/docs/admin/monitoring.md index 7a635a9..f54a585 100644 --- a/docs/admin/monitoring.md +++ b/docs/admin/monitoring.md @@ -3,7 +3,7 @@ ## 대시보드 - **Grafana**: `https://logging.sjanglab.org` (익명 Viewer 접근 가능, wg-admin 경유) -- **Gatus**: `https://gatus.sjanglab.org` (외부 상태 페이지) +- **Gatus**: `https://status.sjanglab.org` (tailnet 내부 공개 상태 페이지) ## 스택 구성 diff --git a/docs/admin/network.md b/docs/admin/network.md index ad254f2..3bd6def 100644 --- a/docs/admin/network.md +++ b/docs/admin/network.md @@ -124,10 +124,10 @@ sequenceDiagram | 호스트 | 외부 개방 포트 | wg-admin 개방 포트 | |--------|--------------|-------------------| -| eta | 80, 443, 10022 (SSH + Rate limiting), 2323 (Upterm relay) | 10022 | +| eta | 80, 443, 10022 (SSH + Rate limiting), 2323 (Upterm relay) | 10022, 8081 (Gatus) | | psi | — | 80/443 (Nixbot upstream), 10022, 5000 (Harmonia), 5432 (Nixbot/PostgreSQL) | | rho | — | 10022, 5432 (PostgreSQL), 3000 (Grafana) | -| tau | — | 10022, 5678 (n8n 웹훅) | +| tau | — | 10022, 5678 (n8n 웹훅), 8000 (Vaultwarden) | ## ACME 인증서 diff --git a/docs/admin/security.md b/docs/admin/security.md index 2b71da2..acdc6cc 100644 --- a/docs/admin/security.md +++ b/docs/admin/security.md @@ -237,4 +237,4 @@ flowchart LR ### 서비스 복구 -systemd가 서비스 실패 시 자동 재시작합니다. 모니터링은 Gatus(`gatus.sjanglab.org`)에서 헬스체크를 수행합니다. +systemd가 서비스 실패 시 자동 재시작합니다. 모니터링은 Gatus(`status.sjanglab.org`)에서 헬스체크를 수행합니다. diff --git a/hosts/eta.nix b/hosts/eta.nix index 82afe80..14f1fa1 100644 --- a/hosts/eta.nix +++ b/hosts/eta.nix @@ -14,7 +14,6 @@ in ../modules/uptermd ../modules/gatus ../modules/monitoring/vector - ../modules/monitoring/reverse-proxy.nix ../modules/n8n/reverse-proxy.nix ../modules/acme/sync.nix ]; @@ -25,36 +24,49 @@ in serviceName = "acme-sync-to-tau"; remoteHost = hosts.tau.wg-admin; } - { - domain = "ollama.sjanglab.org"; - serviceName = "acme-sync-ollama-to-psi"; - remoteUser = "acme-sync-ollama"; - remoteHost = hosts.psi.wg-admin; - } { domain = "docling.sjanglab.org"; serviceName = "acme-sync-docling-to-psi"; remoteHost = hosts.psi.wg-admin; } { - domain = "vllm.sjanglab.org"; - serviceName = "acme-sync-vllm-to-psi"; - remoteUser = "acme-sync-vllm"; - remoteHost = hosts.psi.wg-admin; + domain = "status.sjanglab.org"; + serviceName = "acme-sync-status-to-rho"; + remoteUser = "acme-sync-status"; + remoteHost = hosts.rho.wg-admin; + } + { + domain = "logging.sjanglab.org"; + serviceName = "acme-sync-logging-to-rho"; + remoteUser = "acme-sync-logging"; + remoteHost = hosts.rho.wg-admin; + } + { + domain = "vault.sjanglab.org"; + serviceName = "acme-sync-vaultwarden-to-tau"; + remoteUser = "acme-sync-vaultwarden"; + remoteHost = hosts.tau.wg-admin; } ]; disko.rootDisk = "/dev/vda"; # ACME certificates for internal services - security.acme.certs."ollama.sjanglab.org" = { + security.acme.certs."status.sjanglab.org" = { + dnsProvider = "cloudflare"; + environmentFile = config.sops.secrets.cloudflare-credentials.path; + webroot = null; + group = "acme"; + }; + + security.acme.certs."logging.sjanglab.org" = { dnsProvider = "cloudflare"; environmentFile = config.sops.secrets.cloudflare-credentials.path; webroot = null; group = "acme"; }; - security.acme.certs."vllm.sjanglab.org" = { + security.acme.certs."vault.sjanglab.org" = { dnsProvider = "cloudflare"; environmentFile = config.sops.secrets.cloudflare-credentials.path; webroot = null; diff --git a/hosts/rho.nix b/hosts/rho.nix index 967598e..4d4fa2c 100644 --- a/hosts/rho.nix +++ b/hosts/rho.nix @@ -9,6 +9,8 @@ ../modules/borgbackup/rho/client.nix ../modules/borgbackup/mirror.nix ../modules/monitoring/vector/monitor-systems.nix + ../modules/monitoring/reverse-proxy.nix + ../modules/gatus/reverse-proxy.nix ]; disko.rootDisk = "/dev/disk/by-id/nvme-eui.00000000000000006479a79cdac0038a"; diff --git a/hosts/tau.nix b/hosts/tau.nix index eadbe88..c1bd393 100644 --- a/hosts/tau.nix +++ b/hosts/tau.nix @@ -11,6 +11,7 @@ ../modules/monitoring/vector/monitor-services.nix ../modules/nextcloud ../modules/n8n + ../modules/vaultwarden/reverse-proxy.nix ]; disko.rootDisk = "/dev/disk/by-id/nvme-eui.00000000000000006479a79cdac0038f"; diff --git a/modules/authentik/nginx-locations.nix b/modules/authentik/nginx-locations.nix index 066aff5..2d35bd1 100644 --- a/modules/authentik/nginx-locations.nix +++ b/modules/authentik/nginx-locations.nix @@ -27,8 +27,11 @@ in extraConfig = '' internal; proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header Content-Length 0; + proxy_set_header Host $host; + proxy_set_header X-Original-URL $scheme://$host$request_uri; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Authorization $http_authorization; ''; }; @@ -37,7 +40,10 @@ in "/outpost.goauthentik.io" = { proxyPass = "${authentikOutpost}/outpost.goauthentik.io"; extraConfig = '' - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Original-URL $scheme://$host$request_uri; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Authorization $http_authorization; ''; }; diff --git a/modules/gatus/check.nix b/modules/gatus/check.nix index 343ebdf..574583f 100644 --- a/modules/gatus/check.nix +++ b/modules/gatus/check.nix @@ -18,7 +18,7 @@ }: let cfg = config.gatusCheck; - gatusApi = "https://gatus.sjanglab.org"; + gatusApi = "http://${config.networking.sbee.hosts.eta.wg-admin}:8081"; # Gatus external endpoint key: "${group}_${name}" with spaces/special chars → hyphens mkKey = @@ -135,9 +135,6 @@ in message = "gatusCheck.push '${ep.name}': exactly one of 'url' or 'systemdService' must be set"; }) cfg.push; - # Resolve gatus.sjanglab.org → eta WG IP for hosts behind NAT/WG - networking.hosts.${config.networking.sbee.hosts.eta.wg-admin} = [ "gatus.sjanglab.org" ]; - sops.secrets.gatus-push-token = { sopsFile = ./secrets.yaml; }; diff --git a/modules/gatus/default.nix b/modules/gatus/default.nix index b15270d..4cfb9ff 100644 --- a/modules/gatus/default.nix +++ b/modules/gatus/default.nix @@ -1,29 +1,22 @@ { config, lib, ... }: let inherit (config.networking.sbee) hosts; - domain = "gatus.sjanglab.org"; port = 8081; # 8080 is used by headscale systemCollector = hosts.rho.wg-admin; cfg = config.gatusCheck; in { - imports = [ - ../acme - ./check.nix - ]; + imports = [ ./check.nix ]; services.gatus = { enable = true; environmentFile = config.sops.secrets.gatus-env.path; settings = { - web.port = port; - metrics = true; - - security.basic = { - username = "admin"; - # bcrypt hash → base64 encoded, interpolated from environmentFile - password-bcrypt-base64 = "\${GATUS_SECURITY_BASIC_PASSWORD}"; + web = { + address = "0.0.0.0"; + inherit port; }; + metrics = true; alerting.ntfy = { topic = "gatus"; @@ -63,9 +56,7 @@ in # psi (mkExtEndpoint "Nixbot" "ci") (mkExtEndpoint "Nixbot PostgreSQL" "ci") - (mkExtEndpoint "Ollama" "ai") (mkExtEndpoint "Docling" "ai") - (mkExtEndpoint "vLLM" "ai") # tau (mkExtEndpoint "Nextcloud" "apps") (mkExtEndpoint "n8n" "apps") @@ -82,30 +73,6 @@ in sopsFile = ./secrets.yaml; }; - # ACME certificate (DNS challenge via Cloudflare) - security.acme.certs.${domain} = { - dnsProvider = "cloudflare"; - environmentFile = config.sops.secrets.cloudflare-credentials.path; - webroot = null; - group = "nginx"; - }; - - # Nginx reverse proxy - services.nginx.virtualHosts.${domain} = { - forceSSL = true; - useACMEHost = domain; - locations."/" = { - proxyPass = "http://127.0.0.1:${toString port}"; - proxyWebsockets = true; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - ''; - }; - }; - # Vector: scrape Gatus /metrics → push to rho Prometheus # Independent source+sink pair, does not interfere with existing Vector pipeline services.vector.settings = { @@ -123,8 +90,5 @@ in }; }; - networking.firewall.allowedTCPPorts = [ - 80 - 443 - ]; + networking.firewall.interfaces.wg-admin.allowedTCPPorts = [ port ]; } diff --git a/modules/gatus/reverse-proxy.nix b/modules/gatus/reverse-proxy.nix new file mode 100644 index 0000000..122e5bd --- /dev/null +++ b/modules/gatus/reverse-proxy.nix @@ -0,0 +1,39 @@ +{ config, ... }: +let + inherit (config.networking.sbee) hosts; + domain = "status.sjanglab.org"; + certDir = "/var/lib/acme/${domain}"; +in +{ + imports = [ ../acme/sync.nix ]; + + acmeSyncer.mkReceiver = [ + { + inherit domain; + user = "acme-sync-status"; + } + ]; + + services.nginx = { + enable = true; + + virtualHosts.${domain} = { + forceSSL = true; + sslCertificate = "${certDir}/fullchain.pem"; + sslCertificateKey = "${certDir}/key.pem"; + + locations."/" = { + proxyPass = "http://${hosts.eta.wg-admin}:8081"; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + }; + }; + + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ]; +} diff --git a/modules/headscale/acl-rules.nix b/modules/headscale/acl-rules.nix index 284cb98..f1fb573 100644 --- a/modules/headscale/acl-rules.nix +++ b/modules/headscale/acl-rules.nix @@ -26,6 +26,7 @@ "tag:ai:443" "tag:apps:80" "tag:apps:443" + "tag:monitoring:443" "tag:monitoring:3000" ]; } diff --git a/modules/headscale/default.nix b/modules/headscale/default.nix index e210b8e..407c296 100644 --- a/modules/headscale/default.nix +++ b/modules/headscale/default.nix @@ -56,6 +56,21 @@ in type = "A"; value = "100.64.0.1"; # psi headscale IP } + { + name = "status.sjanglab.org"; + type = "A"; + value = "100.64.0.2"; # rho headscale IP + } + { + name = "logging.sjanglab.org"; + type = "A"; + value = "100.64.0.2"; # rho headscale IP + } + { + name = "vault.sjanglab.org"; + type = "A"; + value = "100.64.0.3"; # tau headscale IP + } { name = "upterm.sjanglab.org"; type = "A"; diff --git a/modules/monitoring/reverse-proxy.nix b/modules/monitoring/reverse-proxy.nix index d22354d..93a9edd 100644 --- a/modules/monitoring/reverse-proxy.nix +++ b/modules/monitoring/reverse-proxy.nix @@ -1,44 +1,54 @@ -# Grafana reverse proxy (deployed on eta) -# Proxies requests to rho where Grafana is running on wg-admin -{ config, ... }: +{ + config, + ... +}: let - inherit (config.networking.sbee) hosts; + inherit (config.networking.sbee) currentHost hosts; + authentikAuth = import ../authentik/nginx-locations.nix { inherit hosts; }; loggingDomain = "logging.sjanglab.org"; + certDir = "/var/lib/acme/${loggingDomain}"; in { - imports = [ ../acme ]; + imports = [ ../acme/sync.nix ]; - services.nginx.virtualHosts.${loggingDomain} = { - forceSSL = true; - useACMEHost = loggingDomain; + acmeSyncer.mkReceiver = [ + { + domain = loggingDomain; + user = "acme-sync-logging"; + } + ]; - locations = { - "/" = { - proxyPass = "http://${hosts.rho.wg-admin}:3000"; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - ''; - }; - # Grafana live (WebSocket) - "/api/live/" = { - proxyPass = "http://${hosts.rho.wg-admin}:3000"; - proxyWebsockets = true; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - ''; + services.nginx = { + enable = true; + + virtualHosts.${loggingDomain} = { + forceSSL = true; + sslCertificate = "${certDir}/fullchain.pem"; + sslCertificateKey = "${certDir}/key.pem"; + + locations = authentikAuth.locations // { + "/" = { + proxyPass = "http://${currentHost.wg-admin}:3000"; + extraConfig = authentikAuth.protectLocation + '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + "/api/live/" = { + proxyPass = "http://${currentHost.wg-admin}:3000"; + proxyWebsockets = true; + extraConfig = authentikAuth.protectLocation + '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; }; }; }; - security.acme.certs.${loggingDomain} = { - dnsProvider = "cloudflare"; - environmentFile = config.sops.secrets.cloudflare-credentials.path; - group = "nginx"; - }; + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ]; } diff --git a/modules/ollama/default.nix b/modules/ollama/default.nix index c34965f..471ebf5 100644 --- a/modules/ollama/default.nix +++ b/modules/ollama/default.nix @@ -94,7 +94,8 @@ in sslCertificate = "${certDir}/fullchain.pem"; sslCertificateKey = "${certDir}/key.pem"; - # Access control: Headscale ACL (network-level, no forward auth) + # API clients do not handle browser redirects from Authentik forward auth. + # Keep application authorization in API credentials when this module is enabled. locations."/" = { proxyPass = "http://127.0.0.1:${toString port}"; recommendedProxySettings = false; # Override Host header manually diff --git a/modules/vaultwarden/default.nix b/modules/vaultwarden/default.nix index 779b569..f817f19 100644 --- a/modules/vaultwarden/default.nix +++ b/modules/vaultwarden/default.nix @@ -1,15 +1,13 @@ { config, ... }: { - imports = [ - ../acme - ../gatus/check.nix - ]; + imports = [ ../gatus/check.nix ]; gatusCheck.pull = [ { name = "Vaultwarden"; - url = "https://vault.sjanglab.org/alive"; + url = "http://127.0.0.1:8000/alive"; group = "apps"; + conditions = [ "[STATUS] == 200" ]; } ]; @@ -35,6 +33,7 @@ # Organization ORG_CREATION_USERS = "sjang.bioe@gmail.com,admin@sjanglab.org"; + ROCKET_ADDRESS = "0.0.0.0"; ROCKET_PORT = 8000; }; }; @@ -46,37 +45,9 @@ mode = "0400"; }; - # ACME certificate (DNS challenge) - security.acme.certs."vault.sjanglab.org" = { - dnsProvider = "cloudflare"; - environmentFile = config.sops.secrets.cloudflare-credentials.path; - webroot = null; - group = "nginx"; - }; - - # Nginx reverse proxy - services.nginx.virtualHosts."vault.sjanglab.org" = { - forceSSL = true; - useACMEHost = "vault.sjanglab.org"; - - locations."/" = { - proxyPass = "http://127.0.0.1:8000"; - proxyWebsockets = true; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - ''; - }; - }; - systemd.tmpfiles.rules = [ "d /var/backup/vaultwarden 0700 vaultwarden vaultwarden -" ]; - networking.firewall.allowedTCPPorts = [ - 80 - 443 - ]; + networking.firewall.interfaces.wg-admin.allowedTCPPorts = [ 8000 ]; } diff --git a/modules/vaultwarden/reverse-proxy.nix b/modules/vaultwarden/reverse-proxy.nix new file mode 100644 index 0000000..d62b965 --- /dev/null +++ b/modules/vaultwarden/reverse-proxy.nix @@ -0,0 +1,39 @@ +{ config, ... }: +let + inherit (config.networking.sbee) hosts; + domain = "vault.sjanglab.org"; + certDir = "/var/lib/acme/${domain}"; +in +{ + imports = [ ../acme/sync.nix ]; + + acmeSyncer.mkReceiver = [ + { + inherit domain; + user = "acme-sync-vaultwarden"; + } + ]; + + services.nginx = { + enable = true; + + virtualHosts.${domain} = { + forceSSL = true; + sslCertificate = "${certDir}/fullchain.pem"; + sslCertificateKey = "${certDir}/key.pem"; + + locations."/" = { + proxyPass = "http://${hosts.eta.wg-admin}:8000"; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + }; + }; + + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ]; +} diff --git a/modules/vllm/default.nix b/modules/vllm/default.nix index fa3a265..b29e453 100644 --- a/modules/vllm/default.nix +++ b/modules/vllm/default.nix @@ -135,6 +135,8 @@ in sslCertificate = "/var/lib/acme/${domain}/fullchain.pem"; sslCertificateKey = "/var/lib/acme/${domain}/key.pem"; + # API clients do not handle browser redirects from Authentik forward auth. + # Keep application authorization in API credentials when this module is enabled. locations."/" = { proxyPass = "http://127.0.0.1:8100"; extraConfig = '' diff --git a/terraform/cloudflare/main.tf b/terraform/cloudflare/main.tf index 45930ef..e7fe720 100644 --- a/terraform/cloudflare/main.tf +++ b/terraform/cloudflare/main.tf @@ -54,16 +54,6 @@ resource "cloudflare_dns_record" "buildbot" { comment = "Nixbot CI/CD edge proxy (eta -> psi)" } -resource "cloudflare_dns_record" "s3" { - zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"] - name = "s3.sjanglab.org" - content = "141.164.53.203" - type = "A" - ttl = 300 - proxied = false - comment = "MinIO S3 API" -} - resource "cloudflare_dns_record" "ntfy" { zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"] name = "ntfy.sjanglab.org" @@ -74,16 +64,6 @@ resource "cloudflare_dns_record" "ntfy" { comment = "ntfy notification service" } -resource "cloudflare_dns_record" "logging" { - zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"] - name = "logging.sjanglab.org" - content = "141.164.53.203" - type = "A" - ttl = 300 - proxied = false - comment = "Grafana logging dashboard" -} - resource "cloudflare_dns_record" "headscale" { zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"] name = "hs.sjanglab.org" @@ -104,26 +84,6 @@ resource "cloudflare_dns_record" "authentik" { comment = "Authentik SSO server" } -resource "cloudflare_dns_record" "vaultwarden" { - zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"] - name = "vault.sjanglab.org" - content = "141.164.53.203" - type = "A" - ttl = 300 - proxied = false - comment = "Vaultwarden password manager" -} - -resource "cloudflare_dns_record" "gatus" { - zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"] - name = "gatus.sjanglab.org" - content = "141.164.53.203" - type = "A" - ttl = 300 - proxied = false - comment = "Gatus status monitoring" -} - resource "cloudflare_dns_record" "n8n" { zone_id = data.sops_file.secrets.data["CLOUDFLARE_ZONE_ID"] name = "n8n.sjanglab.org" diff --git a/terraform/github/repo.tf b/terraform/github/repo.tf index cadb248..dd5410f 100644 --- a/terraform/github/repo.tf +++ b/terraform/github/repo.tf @@ -25,13 +25,11 @@ resource "github_repository" "infra" { allow_update_branch = true web_commit_signoff_required = true - pages { - build_type = "workflow" - source { - branch = "main" - path = "/" - } - } +} + +resource "github_repository_pages" "infra" { + repository = github_repository.infra.name + build_type = "workflow" } resource "github_repository_ruleset" "infra" {