Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/admin/monitoring.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
## 대시보드

- **Grafana**: `https://logging.sjanglab.org` (익명 Viewer 접근 가능, wg-admin 경유)
- **Gatus**: `https://gatus.sjanglab.org` (외부 상태 페이지)
- **Gatus**: `https://status.sjanglab.org` (tailnet 내부 공개 상태 페이지)

## 스택 구성

Expand Down
4 changes: 2 additions & 2 deletions docs/admin/network.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,10 @@ sequenceDiagram

| 호스트 | 외부 개방 포트 | wg-admin 개방 포트 |
|--------|--------------|-------------------|
| eta | 80, 443, 10022 (SSH + Rate limiting), 2323 (Upterm relay) | 10022 |
| eta | 80, 443, 10022 (SSH + Rate limiting), 2323 (Upterm relay) | 10022, 8081 (Gatus) |
| psi | — | 80/443 (Nixbot upstream), 10022, 5000 (Harmonia), 5432 (Nixbot/PostgreSQL) |
| rho | — | 10022, 5432 (PostgreSQL), 3000 (Grafana) |
| tau | — | 10022, 5678 (n8n 웹훅) |
| tau | — | 10022, 5678 (n8n 웹훅), 8000 (Vaultwarden) |

## ACME 인증서

Expand Down
2 changes: 1 addition & 1 deletion docs/admin/security.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,4 +237,4 @@ flowchart LR

### 서비스 복구

systemd가 서비스 실패 시 자동 재시작합니다. 모니터링은 Gatus(`gatus.sjanglab.org`)에서 헬스체크를 수행합니다.
systemd가 서비스 실패 시 자동 재시작합니다. 모니터링은 Gatus(`status.sjanglab.org`)에서 헬스체크를 수행합니다.
38 changes: 25 additions & 13 deletions hosts/eta.nix
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ in
../modules/uptermd
../modules/gatus
../modules/monitoring/vector
../modules/monitoring/reverse-proxy.nix
../modules/n8n/reverse-proxy.nix
../modules/acme/sync.nix
];
Expand All @@ -25,36 +24,49 @@ in
serviceName = "acme-sync-to-tau";
remoteHost = hosts.tau.wg-admin;
}
{
domain = "ollama.sjanglab.org";
serviceName = "acme-sync-ollama-to-psi";
remoteUser = "acme-sync-ollama";
remoteHost = hosts.psi.wg-admin;
}
{
domain = "docling.sjanglab.org";
serviceName = "acme-sync-docling-to-psi";
remoteHost = hosts.psi.wg-admin;
}
{
domain = "vllm.sjanglab.org";
serviceName = "acme-sync-vllm-to-psi";
remoteUser = "acme-sync-vllm";
remoteHost = hosts.psi.wg-admin;
domain = "status.sjanglab.org";
serviceName = "acme-sync-status-to-rho";
remoteUser = "acme-sync-status";
remoteHost = hosts.rho.wg-admin;
}
{
domain = "logging.sjanglab.org";
serviceName = "acme-sync-logging-to-rho";
remoteUser = "acme-sync-logging";
remoteHost = hosts.rho.wg-admin;
}
{
domain = "vault.sjanglab.org";
serviceName = "acme-sync-vaultwarden-to-tau";
remoteUser = "acme-sync-vaultwarden";
remoteHost = hosts.tau.wg-admin;
}
];

disko.rootDisk = "/dev/vda";

# ACME certificates for internal services
security.acme.certs."ollama.sjanglab.org" = {
security.acme.certs."status.sjanglab.org" = {
dnsProvider = "cloudflare";
environmentFile = config.sops.secrets.cloudflare-credentials.path;
webroot = null;
group = "acme";
};

security.acme.certs."logging.sjanglab.org" = {
dnsProvider = "cloudflare";
environmentFile = config.sops.secrets.cloudflare-credentials.path;
webroot = null;
group = "acme";
};

security.acme.certs."vllm.sjanglab.org" = {
security.acme.certs."vault.sjanglab.org" = {
dnsProvider = "cloudflare";
environmentFile = config.sops.secrets.cloudflare-credentials.path;
webroot = null;
Expand Down
2 changes: 2 additions & 0 deletions hosts/rho.nix
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
../modules/borgbackup/rho/client.nix
../modules/borgbackup/mirror.nix
../modules/monitoring/vector/monitor-systems.nix
../modules/monitoring/reverse-proxy.nix
../modules/gatus/reverse-proxy.nix
];

disko.rootDisk = "/dev/disk/by-id/nvme-eui.00000000000000006479a79cdac0038a";
Expand Down
1 change: 1 addition & 0 deletions hosts/tau.nix
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
../modules/monitoring/vector/monitor-services.nix
../modules/nextcloud
../modules/n8n
../modules/vaultwarden/reverse-proxy.nix
];

disko.rootDisk = "/dev/disk/by-id/nvme-eui.00000000000000006479a79cdac0038f";
Expand Down
12 changes: 9 additions & 3 deletions modules/authentik/nginx-locations.nix
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ in
extraConfig = ''
internal;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header Content-Length 0;
proxy_set_header Host $host;
proxy_set_header X-Original-URL $scheme://$host$request_uri;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Authorization $http_authorization;
'';
};
Expand All @@ -37,7 +40,10 @@ in
"/outpost.goauthentik.io" = {
proxyPass = "${authentikOutpost}/outpost.goauthentik.io";
extraConfig = ''
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Original-URL $scheme://$host$request_uri;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Authorization $http_authorization;
'';
};
Expand Down
5 changes: 1 addition & 4 deletions modules/gatus/check.nix
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
}:
let
cfg = config.gatusCheck;
gatusApi = "https://gatus.sjanglab.org";
gatusApi = "http://${config.networking.sbee.hosts.eta.wg-admin}:8081";

# Gatus external endpoint key: "${group}_${name}" with spaces/special chars → hyphens
mkKey =
Expand Down Expand Up @@ -135,9 +135,6 @@ in
message = "gatusCheck.push '${ep.name}': exactly one of 'url' or 'systemdService' must be set";
}) cfg.push;

# Resolve gatus.sjanglab.org → eta WG IP for hosts behind NAT/WG
networking.hosts.${config.networking.sbee.hosts.eta.wg-admin} = [ "gatus.sjanglab.org" ];

sops.secrets.gatus-push-token = {
sopsFile = ./secrets.yaml;
};
Expand Down
48 changes: 6 additions & 42 deletions modules/gatus/default.nix
Original file line number Diff line number Diff line change
@@ -1,29 +1,22 @@
{ config, lib, ... }:
let
inherit (config.networking.sbee) hosts;
domain = "gatus.sjanglab.org";
port = 8081; # 8080 is used by headscale
systemCollector = hosts.rho.wg-admin;
cfg = config.gatusCheck;
in
{
imports = [
../acme
./check.nix
];
imports = [ ./check.nix ];

services.gatus = {
enable = true;
environmentFile = config.sops.secrets.gatus-env.path;
settings = {
web.port = port;
metrics = true;

security.basic = {
username = "admin";
# bcrypt hash → base64 encoded, interpolated from environmentFile
password-bcrypt-base64 = "\${GATUS_SECURITY_BASIC_PASSWORD}";
web = {
address = "0.0.0.0";
inherit port;
};
metrics = true;

alerting.ntfy = {
topic = "gatus";
Expand Down Expand Up @@ -63,9 +56,7 @@ in
# psi
(mkExtEndpoint "Nixbot" "ci")
(mkExtEndpoint "Nixbot PostgreSQL" "ci")
(mkExtEndpoint "Ollama" "ai")
(mkExtEndpoint "Docling" "ai")
(mkExtEndpoint "vLLM" "ai")
# tau
(mkExtEndpoint "Nextcloud" "apps")
(mkExtEndpoint "n8n" "apps")
Expand All @@ -82,30 +73,6 @@ in
sopsFile = ./secrets.yaml;
};

# ACME certificate (DNS challenge via Cloudflare)
security.acme.certs.${domain} = {
dnsProvider = "cloudflare";
environmentFile = config.sops.secrets.cloudflare-credentials.path;
webroot = null;
group = "nginx";
};

# Nginx reverse proxy
services.nginx.virtualHosts.${domain} = {
forceSSL = true;
useACMEHost = domain;
locations."/" = {
proxyPass = "http://127.0.0.1:${toString port}";
proxyWebsockets = true;
extraConfig = ''
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
'';
};
};

# Vector: scrape Gatus /metrics → push to rho Prometheus
# Independent source+sink pair, does not interfere with existing Vector pipeline
services.vector.settings = {
Expand All @@ -123,8 +90,5 @@ in
};
};

networking.firewall.allowedTCPPorts = [
80
443
];
networking.firewall.interfaces.wg-admin.allowedTCPPorts = [ port ];
}
39 changes: 39 additions & 0 deletions modules/gatus/reverse-proxy.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{ config, ... }:
let
inherit (config.networking.sbee) hosts;
domain = "status.sjanglab.org";
certDir = "/var/lib/acme/${domain}";
in
{
imports = [ ../acme/sync.nix ];

acmeSyncer.mkReceiver = [
{
inherit domain;
user = "acme-sync-status";
}
];

services.nginx = {
enable = true;

virtualHosts.${domain} = {
forceSSL = true;
sslCertificate = "${certDir}/fullchain.pem";
sslCertificateKey = "${certDir}/key.pem";

locations."/" = {
proxyPass = "http://${hosts.eta.wg-admin}:8081";
proxyWebsockets = true;
extraConfig = ''
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
'';
};
};
};

networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 443 ];
}
1 change: 1 addition & 0 deletions modules/headscale/acl-rules.nix
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"tag:ai:443"
"tag:apps:80"
"tag:apps:443"
"tag:monitoring:443"
"tag:monitoring:3000"
];
}
Expand Down
15 changes: 15 additions & 0 deletions modules/headscale/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,21 @@ in
type = "A";
value = "100.64.0.1"; # psi headscale IP
}
{
name = "status.sjanglab.org";
type = "A";
value = "100.64.0.2"; # rho headscale IP
}
{
name = "logging.sjanglab.org";
type = "A";
value = "100.64.0.2"; # rho headscale IP
}
{
name = "vault.sjanglab.org";
type = "A";
value = "100.64.0.3"; # tau headscale IP
}
{
name = "upterm.sjanglab.org";
type = "A";
Expand Down
Loading
Loading