From 63e805716f640ee4ec2c25e214a5f366de377d46 Mon Sep 17 00:00:00 2001 From: rajashish147 Date: Fri, 3 Apr 2026 17:13:33 +0530 Subject: [PATCH 1/2] Refactor API deployment and Nginx configuration for Docker integration - Updated deploy.yml to use Docker container names instead of ports for health checks. - Modified Nginx configuration to use container names for upstream servers. - Enhanced rollback.sh to check Nginx configuration using Docker. - Adjusted vps-setup.sh to stop system Nginx and start Docker Nginx for SSL handling. - Improved deploy-bluegreen.sh to ensure active container existence and validate Nginx configuration. - Added health checks and logging improvements for better deployment stability. --- .github/workflows/deploy.yml | 53 +++--- .github/workflows/pr.yml | 2 +- infra/docker-compose.monitoring.yml | 47 +++++- infra/nginx/api.conf | 14 +- scripts/deploy-bluegreen.sh | 245 +++++++++++++++++++--------- scripts/rollback.sh | 4 +- scripts/vps-setup.sh | 25 ++- 7 files changed, 278 insertions(+), 112 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 627e35b..670aec5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -624,23 +624,30 @@ jobs: # Determine active slot (blue/green) ACTIVE_SLOT=$(cat /var/run/api/active-slot 2>/dev/null || echo "blue") - if [ "$ACTIVE_SLOT" = "green" ]; then BACKEND_PORT=3002; else BACKEND_PORT=3001; fi - - echo "=== API Health Gate (slot: $ACTIVE_SLOT, port: $BACKEND_PORT) ===" - - # Poll /ready endpoint (internal readiness probe) + ACTIVE_CONTAINER="api-$ACTIVE_SLOT" + + echo "=== API Health Gate (slot: $ACTIVE_SLOT, container: $ACTIVE_CONTAINER) ===" + + # Guard: container must exist before we try to reach it + docker inspect "$ACTIVE_CONTAINER" >/dev/null 2>&1 || { + echo "❌ Container $ACTIVE_CONTAINER not found" + exit 1 + } + + # Poll /ready via Docker service DNS (no host port binding needed) for i in $(seq 1 15); do - STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:$BACKEND_PORT/ready" 2>/dev/null || echo "000") + STATUS=$(docker exec "$ACTIVE_CONTAINER" \ + curl -s -o /dev/null -w "%{http_code}" "http://localhost:3000/ready" 2>/dev/null || echo "000") if [ "$STATUS" = "200" ]; then - echo "✓ API ready on port $BACKEND_PORT (attempt $i)" + echo "✓ API ready (container $ACTIVE_CONTAINER, attempt $i)" exit 0 fi echo " Attempt $i: HTTP $STATUS — waiting..." sleep 2 done - + echo "❌ API /ready did not return 200 after 30s — monitoring sync would fail anyway" - docker logs "api-$ACTIVE_SLOT" --tail 30 2>/dev/null || true + docker logs "$ACTIVE_CONTAINER" --tail 30 2>/dev/null || true exit 1 # --------------------------------------------------------------------------- @@ -673,11 +680,12 @@ jobs: [ -d "$DEPLOY_ROOT" ] || { echo "❌ DEPLOY_ROOT not found: $DEPLOY_ROOT"; exit 1; } cd "$DEPLOY_ROOT" INFRA_DIR="$DEPLOY_ROOT/infra" - NGINX_LIVE="/etc/nginx/sites-enabled/api.conf" + NGINX_LIVE="$DEPLOY_ROOT/infra/nginx/live/api.conf" + NGINX_BACKUP_DIR="$DEPLOY_ROOT/infra/nginx/backup" ACTIVE_SLOT_FILE="/var/run/api/active-slot" ACTIVE_SLOT=$(cat "$ACTIVE_SLOT_FILE" 2>/dev/null || echo "blue") - if [ "$ACTIVE_SLOT" = "green" ]; then BACKEND_PORT=3002; else BACKEND_PORT=3001; fi + ACTIVE_CONTAINER="api-$ACTIVE_SLOT" # Load env from .env — exports DEPLOY_ROOT, API_HOSTNAME, and all # app variables. DEPLOY_ROOT is already exported above; load-env.sh uses it. @@ -685,22 +693,26 @@ jobs: echo "✓ API_HOSTNAME: $API_HOSTNAME" - echo "=== Syncing Nginx (slot: $ACTIVE_SLOT, port: $BACKEND_PORT) ===" - sudo cp "$NGINX_LIVE" /tmp/api.conf.bak 2>/dev/null || true + # Ensure live/backup dirs exist + mkdir -p "$(dirname "$NGINX_LIVE")" "$NGINX_BACKUP_DIR" + + echo "=== Syncing Nginx (slot: $ACTIVE_SLOT, container: $ACTIVE_CONTAINER) ===" + cp "$NGINX_LIVE" "$NGINX_BACKUP_DIR/api.conf.bak.$(date +%s)" 2>/dev/null || true NGINX_TMP=$(mktemp /tmp/fieldtrack-nginx.XXXXXX.conf) sed \ - -e "s|__BACKEND_PORT__|$BACKEND_PORT|g" \ + -e "s|__ACTIVE_CONTAINER__|$ACTIVE_CONTAINER|g" \ -e "s|__API_HOSTNAME__|$API_HOSTNAME|g" \ "$INFRA_DIR/nginx/api.conf" > "$NGINX_TMP" - sudo cp "$NGINX_TMP" "$NGINX_LIVE" + cp "$NGINX_TMP" "$NGINX_LIVE" rm -f "$NGINX_TMP" - if ! sudo nginx -t 2>&1; then + if ! docker exec nginx nginx -t 2>&1; then echo "Nginx test failed — restoring backup..." - sudo cp /tmp/api.conf.bak "$NGINX_LIVE" + LATEST_BAK=$(ls -1t "$NGINX_BACKUP_DIR"/api.conf.bak.* 2>/dev/null | head -1 || true) + [ -n "$LATEST_BAK" ] && cp "$LATEST_BAK" "$NGINX_LIVE" exit 1 fi - sudo systemctl reload nginx + docker exec nginx nginx -s reload echo "✓ Nginx reloaded." # ROUTING VALIDATION — Test actual traffic through Nginx @@ -718,8 +730,9 @@ jobs: echo "✓ Nginx routing verified (HTTP $ROUTE_STATUS)" else echo "❌ Nginx routing broken (HTTP $ROUTE_STATUS expected 200) — restoring backup..." - sudo cp /tmp/api.conf.bak "$NGINX_LIVE" - sudo nginx -t 2>&1 && sudo systemctl reload nginx || true + LATEST_BAK=$(ls -1t "$NGINX_BACKUP_DIR"/api.conf.bak.* 2>/dev/null | head -1 || true) + [ -n "$LATEST_BAK" ] && cp "$LATEST_BAK" "$NGINX_LIVE" + docker exec nginx nginx -t 2>&1 && docker exec nginx nginx -s reload || true exit 1 fi diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e90e8fd..1f82df4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -241,7 +241,7 @@ jobs: if: needs.detect-changes.outputs.infra == 'true' run: | sed \ - -e 's/__BACKEND_PORT__/3001/g' \ + -e 's/__ACTIVE_CONTAINER__/api-blue/g' \ -e 's/__API_HOSTNAME__/api.test.local/g' \ infra/nginx/api.conf > /tmp/nginx.conf diff --git a/infra/docker-compose.monitoring.yml b/infra/docker-compose.monitoring.yml index 095772b..133fef6 100644 --- a/infra/docker-compose.monitoring.yml +++ b/infra/docker-compose.monitoring.yml @@ -94,8 +94,8 @@ services: image: prom/prometheus:v2.52.0 container_name: prometheus restart: unless-stopped - ports: - - "127.0.0.1:9090:9090" + expose: + - "9090" environment: - METRICS_SCRAPE_TOKEN=${METRICS_SCRAPE_TOKEN} @@ -203,6 +203,49 @@ services: max-size: "10m" max-file: "3" + nginx: + image: nginx:1.25-alpine + container_name: nginx + restart: unless-stopped + ports: + - "80:80" + - "443:443" + + volumes: + # Rendered nginx config — written by deploy script on each deploy + - ./nginx/live:/etc/nginx/conf.d:ro + # SSL certificates (managed by certbot on the host) + - /etc/ssl/api:/etc/ssl/api:ro + # ACME challenge webroot for certbot renewal + - /var/www/certbot:/var/www/certbot:ro + # Nginx access logs shared with promtail + - /var/log/nginx:/var/log/nginx + + networks: + - api_network + + depends_on: + grafana: + condition: service_healthy + + deploy: + resources: + limits: + memory: 64m + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:80/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + networks: api_network: external: true diff --git a/infra/nginx/api.conf b/infra/nginx/api.conf index df8aa21..6a13eef 100644 --- a/infra/nginx/api.conf +++ b/infra/nginx/api.conf @@ -8,7 +8,7 @@ map $http_upgrade $connection_upgrade { } upstream api_backend { - server 127.0.0.1:__BACKEND_PORT__ max_fails=3 fail_timeout=30s; + server __ACTIVE_CONTAINER__:3000 max_fails=3 fail_timeout=30s; keepalive 32; } @@ -73,12 +73,20 @@ geo $realip_remote_addr $is_trusted_source { 198.41.128.0/17 1; } -# HTTP → HTTPS +# HTTP → HTTPS (with ACME challenge passthrough for certbot renewal) server { listen 80; listen [::]:80; server_name __API_HOSTNAME__; - return 301 https://$host$request_uri; + + # Let certbot serve ACME challenges for certificate renewal + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } } # HTTPS SERVER diff --git a/scripts/deploy-bluegreen.sh b/scripts/deploy-bluegreen.sh index a9278ce..ba28df1 100644 --- a/scripts/deploy-bluegreen.sh +++ b/scripts/deploy-bluegreen.sh @@ -100,7 +100,7 @@ _ft_snapshot() { { set +x; } 2>/dev/null printf '[DEPLOY] -- SYSTEM SNAPSHOT ----------------------------------------\n' >&2 printf '[DEPLOY] slot_file = %s\n' "$(cat "${ACTIVE_SLOT_FILE:-/var/run/api/active-slot}" 2>/dev/null || echo 'MISSING')" >&2 - printf '[DEPLOY] nginx_port = %s\n' "$(grep -oP 'server 127\.0\.0\.1:\K[0-9]+' "${NGINX_CONF:-/etc/nginx/sites-enabled/api.conf}" 2>/dev/null | head -1 || echo 'unreadable')" >&2 + printf '[DEPLOY] nginx_upstream = %s\n' "$(grep -oE 'server (api-blue|api-green):3000' "${NGINX_CONF:-$HOME/api/infra/nginx/live/api.conf}" 2>/dev/null | head -1 || echo 'unreadable')" >&2 printf '[DEPLOY] containers =\n' >&2 docker ps --format '[DEPLOY] {{.Names}} -> {{.Status}} ({{.Ports}})' 1>&2 2>/dev/null \ || printf '[DEPLOY] (docker ps unavailable)\n' >&2 @@ -155,8 +155,6 @@ IMAGE_SHA="${1:-latest}" BLUE_NAME="api-blue" GREEN_NAME="api-green" -BLUE_PORT=3001 -GREEN_PORT=3002 APP_PORT=3000 NETWORK="api_network" @@ -172,7 +170,9 @@ REPO_DIR="$DEPLOY_ROOT" SLOT_DIR="/var/run/api" ACTIVE_SLOT_FILE="$SLOT_DIR/active-slot" -NGINX_CONF="/etc/nginx/sites-enabled/api.conf" +NGINX_CONF="$REPO_DIR/infra/nginx/live/api.conf" +NGINX_LIVE_DIR="$REPO_DIR/infra/nginx/live" +NGINX_BACKUP_DIR="$REPO_DIR/infra/nginx/backup" NGINX_TEMPLATE="$REPO_DIR/infra/nginx/api.conf" MAX_HISTORY=5 MAX_HEALTH_ATTEMPTS=40 @@ -253,6 +253,29 @@ _ft_check_external_ready() { return 1 } +# --------------------------------------------------------------------------- +# RETRY CURL -- wraps curl -sf with retries + 1s backoff +# _ft_retry_curl [max_attempts=10] [extra curl flags...] +# Returns 0 on first 2xx success, 1 after all attempts exhausted. +# --------------------------------------------------------------------------- +_ft_retry_curl() { + { set +x; } 2>/dev/null + local url="$1" + local max="${2:-10}" + shift 2 || shift $# + local i=0 + while [ "$i" -lt "$max" ]; do + i=$((i + 1)) + if curl -sf --max-time 5 "$@" "$url" >/dev/null 2>&1; then + set -x + return 0 + fi + sleep 1 + done + set -x + return 1 +} + # --------------------------------------------------------------------------- # SLOT DIRECTORY AND FILE MANAGEMENT # --------------------------------------------------------------------------- @@ -342,16 +365,16 @@ _ft_resolve_slot() { recovered_slot="green" _ft_log "msg='recovery: only green running' slot=green" elif [ "$blue_running" = "true" ] && [ "$green_running" = "true" ]; then - # Both running -- read nginx upstream port as authoritative tiebreaker. - local nginx_port - nginx_port=$(grep -oP 'server 127\.0\.0\.1:\K[0-9]+' "$NGINX_CONF" 2>/dev/null | head -1 || echo "") - if [ "$nginx_port" = "$BLUE_PORT" ]; then recovered_slot="blue" - elif [ "$nginx_port" = "$GREEN_PORT" ]; then recovered_slot="green" + # Both running -- read nginx upstream container as authoritative tiebreaker. + local nginx_upstream + nginx_upstream=$(grep -oE 'server (api-blue|api-green):3000' "$NGINX_CONF" 2>/dev/null | grep -oE 'api-blue|api-green' | head -1 || echo "") + if [ "$nginx_upstream" = "api-blue" ]; then recovered_slot="blue" + elif [ "$nginx_upstream" = "api-green" ]; then recovered_slot="green" else recovered_slot="blue" - _ft_log "level=WARN msg='both containers running and nginx port ambiguous, defaulting to blue' nginx_port=${nginx_port}" + _ft_log "level=WARN msg='both containers running and nginx upstream ambiguous, defaulting to blue' nginx_upstream=${nginx_upstream}" fi - _ft_log "msg='recovery: both containers running, nginx tiebreaker' nginx_port=${nginx_port} slot=${recovered_slot}" + _ft_log "msg='recovery: both containers running, nginx tiebreaker' nginx_upstream=${nginx_upstream} slot=${recovered_slot}" else # Neither running -- first deploy. recovered_slot="green" @@ -403,6 +426,25 @@ chmod 600 "$DEPLOY_ROOT/infra/.env.monitoring" 2>/dev/null || true _ft_log "msg='env contract validated'" +# NGINX CONTAINER GUARD -- nginx MUST run as a Docker container on api_network. +# With container-name upstreams (server api-blue:3000), Docker's embedded DNS +# (127.0.0.11) is required for name resolution. This only works from WITHIN +# Docker containers on the same network -- not from a host systemd nginx service. +if ! docker inspect nginx >/dev/null 2>&1; then + _ft_log "level=ERROR msg='nginx container not found -- nginx must run as Docker container on api_network. Run: docker compose --env-file infra/.env.monitoring -f infra/docker-compose.monitoring.yml up -d nginx'" + _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=nginx_container_missing" +fi +_NGINX_NETWORK=$(docker inspect nginx --format='{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' 2>/dev/null || echo "") +if ! echo "$_NGINX_NETWORK" | grep -q "$NETWORK"; then + _ft_log "level=ERROR msg='nginx container not on api_network -- container DNS will fail' networks=${_NGINX_NETWORK}" + _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=nginx_not_on_api_network networks=${_NGINX_NETWORK}" +fi +unset _NGINX_NETWORK +_ft_log "msg='nginx container guard passed' container=nginx network=$NETWORK" + +# Ensure nginx live and backup directories exist (deploy user owns them) +mkdir -p "$NGINX_LIVE_DIR" "$NGINX_BACKUP_DIR" + # --------------------------------------------------------------------------- # PREFLIGHT CHECK (policy=warn: missing preflight logs a warning, does not abort) # --------------------------------------------------------------------------- @@ -450,14 +492,14 @@ ACTIVE=$(printf '%s' "$ACTIVE" | tr -d '[:space:]') _ft_validate_slot "$ACTIVE" || exit 1 if [ "$ACTIVE" = "blue" ]; then - ACTIVE_NAME=$BLUE_NAME; ACTIVE_PORT=$BLUE_PORT - INACTIVE="green"; INACTIVE_NAME=$GREEN_NAME; INACTIVE_PORT=$GREEN_PORT + ACTIVE_NAME=$BLUE_NAME + INACTIVE="green"; INACTIVE_NAME=$GREEN_NAME else - ACTIVE_NAME=$GREEN_NAME; ACTIVE_PORT=$GREEN_PORT - INACTIVE="blue"; INACTIVE_NAME=$BLUE_NAME; INACTIVE_PORT=$BLUE_PORT + ACTIVE_NAME=$GREEN_NAME + INACTIVE="blue"; INACTIVE_NAME=$BLUE_NAME fi -_ft_log "msg='slot resolved' active=$ACTIVE active_port=$ACTIVE_PORT inactive=$INACTIVE inactive_port=$INACTIVE_PORT" +_ft_log "msg='slot resolved' active=$ACTIVE active_name=$ACTIVE_NAME inactive=$INACTIVE inactive_name=$INACTIVE_NAME" # --------------------------------------------------------------------------- # INITIAL DEPLOYMENT DETECTION -- no containers exist yet @@ -467,6 +509,21 @@ if ! docker ps -a --format '{{.Names}}' | grep -Eq '^api-(blue|green)$'; then INITIAL_DEPLOY=true fi +# --------------------------------------------------------------------------- +# ACTIVE CONTAINER EXISTENCE GUARD +# Protect against race: active slot file says "blue" but container doesn't exist. +# This catches crash/OOM scenarios before any deploy logic runs. +# --------------------------------------------------------------------------- +if docker ps -a --format '{{.Names}}' | grep -q "^${ACTIVE_NAME}$"; then + if ! docker inspect "$ACTIVE_NAME" >/dev/null 2>&1; then + _ft_log "level=ERROR msg='active container listed by docker ps but inspect failed -- possible race' container=$ACTIVE_NAME" + _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=active_container_inspect_race container=$ACTIVE_NAME" + fi + _ft_log "msg='active container existence guard passed' container=$ACTIVE_NAME" +else + _ft_log "level=WARN msg='active container not running (first deploy or crash recovery)' container=$ACTIVE_NAME" +fi + # --------------------------------------------------------------------------- # IDEMPOTENCY GUARD -- skip deploy if this exact SHA is already the active container # --------------------------------------------------------------------------- @@ -477,7 +534,7 @@ if [ "$_RUNNING_IMAGE" = "$IMAGE" ]; then # SHA matches -- only skip if the active container is also healthy. # If it is unhealthy, proceed so the deploy restarts it cleanly. _IDEMPOTENT_HEALTH=$(timeout 4 curl -s --max-time 3 \ - "http://127.0.0.1:$ACTIVE_PORT/ready" 2>/dev/null || echo "") + "http://$ACTIVE_NAME:$APP_PORT/ready" 2>/dev/null || echo "") if echo "$_IDEMPOTENT_HEALTH" | grep -q '"status":"ready"' 2>/dev/null; then _ft_log "msg='target SHA already running and healthy -- nothing to do' container=$ACTIVE_NAME image=$IMAGE" _ft_exit 0 "DEPLOY_SUCCESS" "reason=idempotent_noop sha=$IMAGE_SHA container=$ACTIVE_NAME" @@ -493,7 +550,7 @@ if [ "$_RUNNING_IMAGE" = "$IMAGE" ]; then # --------------------------------------------------------------------------- # [3/7] START INACTIVE CONTAINER # --------------------------------------------------------------------------- -_ft_state "START_INACTIVE" "msg='starting inactive container' name=$INACTIVE_NAME port=$INACTIVE_PORT" +_ft_state "START_INACTIVE" "msg='starting inactive container' name=$INACTIVE_NAME" if docker ps -a --format '{{.Names}}' | grep -Eq "^${INACTIVE_NAME}$"; then _ft_log "msg='removing stale container' name=$INACTIVE_NAME" @@ -504,7 +561,6 @@ fi timeout 60 docker run -d \ --name "$INACTIVE_NAME" \ --network "$NETWORK" \ - -p "127.0.0.1:$INACTIVE_PORT:$APP_PORT" \ --restart unless-stopped \ --label "api.sha=$IMAGE_SHA" \ --label "api.slot=$INACTIVE" \ @@ -512,7 +568,7 @@ timeout 60 docker run -d \ --env-file "$ENV_FILE" \ "$IMAGE" -_ft_log "msg='container started' name=$INACTIVE_NAME port=$INACTIVE_PORT" +_ft_log "msg='container started' name=$INACTIVE_NAME" # IMAGE IMMUTABILITY CHECK -- confirm running container image matches target SHA. _ACTUAL_IMAGE=$(docker inspect --format '{{.Config.Image}}' "$INACTIVE_NAME" 2>/dev/null || echo "") @@ -541,30 +597,29 @@ _CONN_ATTEMPTS=0 _CONN_OK=false while [ "$_CONN_ATTEMPTS" -lt 5 ]; do _CONN_ATTEMPTS=$((_CONN_ATTEMPTS + 1)) - if timeout 3 curl -s -o /dev/null -w '%{http_code}' \ - "http://127.0.0.1:$INACTIVE_PORT/health" 2>/dev/null | grep -qE '^[0-9]+$'; then + if timeout 3 curl -sf "http://$INACTIVE_NAME:$APP_PORT/health" >/dev/null 2>&1; then _CONN_OK=true break fi - _ft_log "msg='connectivity pre-check waiting' attempt=$_CONN_ATTEMPTS/5 port=$INACTIVE_PORT" - sleep 2 + _ft_log "msg='connectivity pre-check waiting' attempt=$_CONN_ATTEMPTS/5 container=$INACTIVE_NAME" + sleep $((RANDOM % 3 + 1)) done if [ "$_CONN_OK" = "false" ]; then - _ft_log "level=ERROR msg='container port not reachable after connectivity pre-check' port=$INACTIVE_PORT" + _ft_log "level=ERROR msg='container not reachable after connectivity pre-check' container=$INACTIVE_NAME" docker logs "$INACTIVE_NAME" --tail 100 >&2 || true docker stop --time 10 "$INACTIVE_NAME" 2>/dev/null || true docker rm "$INACTIVE_NAME" || true _ft_log "msg='active container still serving -- deploy failed non-destructively' container=$ACTIVE_NAME" - _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=container_port_not_reachable port=$INACTIVE_PORT" + _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=container_not_reachable container=$INACTIVE_NAME" fi unset _CONN_ATTEMPTS _CONN_OK -_ft_log "msg='connectivity pre-check passed' port=$INACTIVE_PORT" +_ft_log "msg='connectivity pre-check passed' container=$INACTIVE_NAME" ATTEMPT=0 until true; do ATTEMPT=$((ATTEMPT + 1)) STATUS=$(timeout 5 curl --max-time 4 -s -o /dev/null -w "%{http_code}" \ - "http://127.0.0.1:$INACTIVE_PORT${HEALTH_ENDPOINT}" || echo "000") + "http://$INACTIVE_NAME:$APP_PORT${HEALTH_ENDPOINT}" || echo "000") if [ "$STATUS" = "200" ]; then _ft_log "msg='internal health check passed' endpoint=$HEALTH_ENDPOINT attempts=$ATTEMPT" @@ -581,7 +636,7 @@ until true; do fi if [ "$ATTEMPT" -ge "$MAX_HEALTH_ATTEMPTS" ]; then - _ft_log "level=ERROR msg='internal health check timed out' attempts=$ATTEMPT status=$STATUS endpoint=http://127.0.0.1:$INACTIVE_PORT${HEALTH_ENDPOINT}" + _ft_log "level=ERROR msg='internal health check timed out' attempts=$ATTEMPT status=$STATUS endpoint=http://$INACTIVE_NAME:$APP_PORT${HEALTH_ENDPOINT}" docker logs "$INACTIVE_NAME" --tail 100 >&2 || true docker stop --time 10 "$INACTIVE_NAME" 2>/dev/null || true docker rm "$INACTIVE_NAME" || true @@ -597,44 +652,58 @@ done # --------------------------------------------------------------------------- # [5/7] SWITCH NGINX UPSTREAM # --------------------------------------------------------------------------- -_ft_state "SWITCH_NGINX" "msg='switching nginx upstream' port=$INACTIVE_PORT" +_ft_state "SWITCH_NGINX" "msg='switching nginx upstream' container=$INACTIVE_NAME" + +# Deterministic stabilization window: give the new container a moment before +# switching nginx (complements the jitter already in the health check loop). +sleep 2 # Backup goes to /etc/nginx/ (NOT sites-enabled/) so nginx does not parse it # during validation and trigger a duplicate-upstream error. NGINX_BACKUP="/etc/nginx/api.conf.bak.$(date +%s)" NGINX_TMP="$(mktemp /tmp/api-nginx.XXXXXX.conf)" +# PRE-RELOAD GATE: confirm container is still ready before pointing nginx at it +if ! timeout 4 curl -sf "http://$INACTIVE_NAME:$APP_PORT/ready" >/dev/null 2>&1; then + _ft_log "level=ERROR msg='pre-reload gate failed: container not ready, aborting nginx reload' container=$INACTIVE_NAME" + docker logs "$INACTIVE_NAME" --tail 50 >&2 || true + docker stop --time 10 "$INACTIVE_NAME" 2>/dev/null || true + docker rm "$INACTIVE_NAME" || true + _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=pre_reload_gate_failed container=$INACTIVE_NAME" +fi +_ft_log "msg='pre-reload gate passed' container=$INACTIVE_NAME" + sed \ - -e "s|__BACKEND_PORT__|$INACTIVE_PORT|g" \ + -e "s|__ACTIVE_CONTAINER__|$INACTIVE_NAME|g" \ -e "s|__API_HOSTNAME__|$API_HOSTNAME|g" \ "$NGINX_TEMPLATE" > "$NGINX_TMP" -sudo cp "$NGINX_CONF" "$NGINX_BACKUP" -sudo cp "$NGINX_TMP" "$NGINX_CONF" +cp "$NGINX_CONF" "$NGINX_BACKUP" +cp "$NGINX_TMP" "$NGINX_CONF" rm -f "$NGINX_TMP" -# Remove stale backups accidentally left in sites-enabled/ by old deploy runs. -sudo rm -f /etc/nginx/sites-enabled/api.conf.bak.* +# Prune old backups (keep last 5) to avoid unbounded growth +ls -1t "$NGINX_BACKUP_DIR"/api.conf.bak.* 2>/dev/null | tail -n +6 | xargs rm -f 2>/dev/null || true -if ! sudo nginx -t 2>&1; then +if ! docker exec nginx nginx -t 2>&1; then _ft_log "level=ERROR msg='nginx config test failed -- restoring backup'" - sudo cp "$NGINX_BACKUP" "$NGINX_CONF" + cp "$NGINX_BACKUP" "$NGINX_CONF" _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=nginx_config_test_failed" fi -sudo systemctl reload nginx -_ft_log "msg='nginx reloaded' upstream=127.0.0.1:$INACTIVE_PORT" +docker exec nginx nginx -s reload +_ft_log "msg='nginx reloaded' upstream=$INACTIVE_NAME:$APP_PORT" -# Upstream sanity check -- confirm nginx config actually points at the new port. +# Upstream sanity check -- confirm nginx config actually points at the new container. # Catches template substitution failures before traffic is affected. -_RELOAD_PORT=$(sudo grep -oE '127\.0\.0\.1:[0-9]+' "$NGINX_CONF" 2>/dev/null | head -1 | cut -d: -f2 || echo "") -if [ "$_RELOAD_PORT" != "$INACTIVE_PORT" ]; then - _ft_log "level=ERROR msg='nginx upstream sanity check failed after reload' expected=$INACTIVE_PORT actual=${_RELOAD_PORT:-unreadable}" - sudo cp "$NGINX_BACKUP" "$NGINX_CONF" - sudo nginx -t 2>&1 && sudo systemctl reload nginx || true - _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=nginx_upstream_mismatch expected=$INACTIVE_PORT actual=${_RELOAD_PORT:-unreadable}" +_RELOAD_CONTAINER=$(grep -oE 'server (api-blue|api-green):3000' "$NGINX_CONF" 2>/dev/null | grep -oE 'api-blue|api-green' | head -1 || echo "") +if [ "$_RELOAD_CONTAINER" != "$INACTIVE_NAME" ]; then + _ft_log "level=ERROR msg='nginx upstream sanity check failed after reload' expected=$INACTIVE_NAME actual=${_RELOAD_CONTAINER:-unreadable}" + cp "$NGINX_BACKUP" "$NGINX_CONF" + docker exec nginx nginx -t 2>&1 && docker exec nginx nginx -s reload || true + _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=nginx_upstream_mismatch expected=$INACTIVE_NAME actual=${_RELOAD_CONTAINER:-unreadable}" fi -unset _RELOAD_PORT -_ft_log "msg='nginx upstream sanity check passed' port=$INACTIVE_PORT" +unset _RELOAD_CONTAINER +_ft_log "msg='nginx upstream sanity check passed' container=$INACTIVE_NAME" # Write the slot file AFTER nginx reload so it always reflects what nginx # is currently serving. If the public health check then fails and we roll @@ -644,12 +713,32 @@ _ft_write_slot "$INACTIVE" # Small settle window to stabilize TLS/keep-alive/edge cases sleep 2 +# POST-SWITCH ROUTING VERIFICATION +# Tests: nginx HTTPS stack is responding + can reach the new container. +# Uses 127.0.0.1 to avoid Cloudflare IP allowlist (same pattern as public check). +_ft_log "msg='post-switch nginx routing verification'" +if ! _ft_retry_curl "https://127.0.0.1/health" 5 --insecure; then + _ft_log "level=ERROR msg='post-switch nginx routing verification failed -- nginx cannot reach new container'" + _ft_snapshot + cp "$NGINX_BACKUP" "$NGINX_CONF" + if docker exec nginx nginx -t 2>&1 && docker exec nginx nginx -s reload; then + _ft_log "msg='nginx restored (post-switch routing failure)'" + else + _ft_log "level=ERROR msg='nginx restore failed during post-switch rollback'" + fi + _ft_write_slot "$ACTIVE" + docker stop --time 10 "$INACTIVE_NAME" 2>/dev/null || true + docker rm "$INACTIVE_NAME" || true + _ft_exit 1 "DEPLOY_FAILED_SAFE" "reason=post_switch_routing_failed container=$INACTIVE_NAME" +fi +_ft_log "msg='post-switch routing verification passed'" + # --------------------------------------------------------------------------- # [6/7] PUBLIC HEALTH CHECK (end-to-end nginx routing) # Validates: # 1. HTTP 200 -- nginx routing, TLS, Host header matching # 2. Body "status":"ready" -- backend /ready endpoint, external services -# 3. Port alignment -- live nginx config points at $INACTIVE_PORT +# 3. Container alignment -- live nginx config points at $INACTIVE_NAME # # NOTE: Uses localhost (127.0.0.1) + Host header to validate nginx routing # while avoiding Cloudflare IP allowlist block (see _ft_check_external_ready). @@ -680,10 +769,10 @@ for _attempt in 1 2 3 4 5; do sleep 5 done -# Port alignment check -- live nginx config MUST point at the new slot's port. -_NGINX_PORT=$(grep -oP 'server 127\.0\.0\.1:\K[0-9]+' "$NGINX_CONF" 2>/dev/null | head -1 || echo "") -if [ -n "$_NGINX_PORT" ] && [ "$_NGINX_PORT" != "$INACTIVE_PORT" ]; then - _ft_log "level=ERROR msg='nginx port mismatch -- slot switch did not take effect' expected=$INACTIVE_PORT actual=$_NGINX_PORT" +# Container alignment check -- live nginx config MUST point at the new container. +_NGINX_CONTAINER=$(grep -oE 'server (api-blue|api-green):3000' "$NGINX_CONF" 2>/dev/null | grep -oE 'api-blue|api-green' | head -1 || echo "") +if [ -n "$_NGINX_CONTAINER" ] && [ "$_NGINX_CONTAINER" != "$INACTIVE_NAME" ]; then + _ft_log "level=ERROR msg='nginx container mismatch -- slot switch did not take effect' expected=$INACTIVE_NAME actual=$_NGINX_CONTAINER" _PUB_PASSED=false fi @@ -692,8 +781,8 @@ if [ "$_PUB_PASSED" != "true" ]; then _ft_snapshot _ft_log "msg='restoring previous nginx config'" - sudo cp "$NGINX_BACKUP" "$NGINX_CONF" - if sudo nginx -t 2>&1 && sudo systemctl reload nginx; then + cp "$NGINX_BACKUP" "$NGINX_CONF" + if docker exec nginx nginx -t 2>&1 && docker exec nginx nginx -s reload; then _ft_log "msg='nginx restored to previous config'" else _ft_log "level=ERROR msg='nginx restore failed -- check manually'" @@ -704,11 +793,11 @@ if [ "$_PUB_PASSED" != "true" ]; then docker stop --time 10 "$INACTIVE_NAME" 2>/dev/null || true docker rm "$INACTIVE_NAME" || true - unset _PUB_PASSED _attempt _PUB_STATUS _PUB_BODY _NGINX_PORT + unset _PUB_PASSED _attempt _PUB_STATUS _PUB_BODY _NGINX_CONTAINER if docker ps --format '{{.Names}}' | grep -q "^${ACTIVE_NAME}$"; then _ACTIVE_HEALTH=$(timeout 4 curl -s --max-time 3 \ - "http://127.0.0.1:$ACTIVE_PORT/ready" 2>/dev/null || echo "") + "http://$ACTIVE_NAME:$APP_PORT/ready" 2>/dev/null || echo "") if echo "$_ACTIVE_HEALTH" | grep -q '"status":"ready"' 2>/dev/null; then _ft_log "msg='deploy failed but active container healthy -- skipping rollback' container=$ACTIVE_NAME" unset _ACTIVE_HEALTH @@ -734,8 +823,8 @@ if [ "$_PUB_PASSED" != "true" ]; then fi fi -unset _PUB_PASSED _attempt _PUB_STATUS _PUB_BODY _NGINX_PORT -_ft_log "msg='public health check passed' port=$INACTIVE_PORT host=$API_HOSTNAME endpoint=/ready" +unset _PUB_PASSED _attempt _PUB_STATUS _PUB_BODY _NGINX_CONTAINER +_ft_log "msg='public health check passed' container=$INACTIVE_NAME host=$API_HOSTNAME endpoint=/ready" # --------------------------------------------------------------------------- # [6.5/7] STABILITY CHECK -- re-verify external endpoint after a settle window @@ -756,8 +845,8 @@ if [ "$_STABLE" = "false" ]; then # Restore nginx + slot _ft_log "msg='restoring previous nginx config (stability failure)'" - sudo cp "$NGINX_BACKUP" "$NGINX_CONF" - if sudo nginx -t 2>&1 && sudo systemctl reload nginx; then + cp "$NGINX_BACKUP" "$NGINX_CONF" + if docker exec nginx nginx -t 2>&1 && docker exec nginx nginx -s reload; then _ft_log "msg='nginx restored (stability failure)'" else _ft_log "level=ERROR msg='nginx restore failed during stability rollback -- check manually'" @@ -768,7 +857,7 @@ if [ "$_STABLE" = "false" ]; then if docker ps --format '{{.Names}}' | grep -q "^${ACTIVE_NAME}$"; then _ACTIVE_HEALTH=$(timeout 4 curl -s --max-time 3 \ - "http://127.0.0.1:$ACTIVE_PORT/ready" 2>/dev/null || echo "") + "http://$ACTIVE_NAME:$APP_PORT/ready" 2>/dev/null || echo "") if echo "$_ACTIVE_HEALTH" | grep -q '"status":"ready"' 2>/dev/null; then _ft_log "msg='active container healthy after stability failure -- skipping rollback' container=$ACTIVE_NAME" unset _ACTIVE_HEALTH @@ -815,7 +904,7 @@ else _ft_log "msg='cleanup skipped (first deploy scenario or container already removed)'" fi -_ft_state "SUCCESS" "msg='deployment complete' container=$INACTIVE_NAME sha=$IMAGE_SHA slot=$INACTIVE port=$INACTIVE_PORT" +_ft_state "SUCCESS" "msg='deployment complete' container=$INACTIVE_NAME sha=$IMAGE_SHA slot=$INACTIVE" # --------------------------------------------------------------------------- # FINAL TRUTH CHECK -- verify state matches deployment intent @@ -838,34 +927,34 @@ else _FT_TRUTH_CHECK_PASSED=false fi -# (2) Verify nginx upstream port matches target -_NGINX_PORT=$(grep -oP 'server 127\.0\.0\.1:\K[0-9]+' "$NGINX_CONF" 2>/dev/null | head -1 || echo "") -if [ -n "$_NGINX_PORT" ]; then - if [ "$_NGINX_PORT" != "$INACTIVE_PORT" ]; then - _ft_log "level=ERROR msg='truth check failed: nginx port mismatch' expected=$INACTIVE_PORT actual=$_NGINX_PORT" +# (2) Verify nginx upstream container matches target +_NGINX_CONTAINER=$(grep -oE 'server (api-blue|api-green):3000' "$NGINX_CONF" 2>/dev/null | grep -oE 'api-blue|api-green' | head -1 || echo "") +if [ -n "$_NGINX_CONTAINER" ]; then + if [ "$_NGINX_CONTAINER" != "$INACTIVE_NAME" ]; then + _ft_log "level=ERROR msg='truth check failed: nginx container mismatch' expected=$INACTIVE_NAME actual=$_NGINX_CONTAINER" _FT_TRUTH_CHECK_PASSED=false else - _ft_log "msg='truth check: nginx port correct' port=$_NGINX_PORT" + _ft_log "msg='truth check: nginx upstream correct' container=$_NGINX_CONTAINER" fi else - _ft_log "level=WARN msg='truth check: could not read nginx port'" + _ft_log "level=WARN msg='truth check: could not read nginx upstream'" fi # (3) Compare internal vs external endpoint health -# Internal: direct container endpoint (127.0.0.1:$INACTIVE_PORT/ready) +# Internal: direct container endpoint (http://$INACTIVE_NAME:$APP_PORT/ready) # External: production DNS/Cloudflare (https://$API_HOSTNAME/ready) # Mismatch indicates routing, TLS, or proxy issues if command -v curl >/dev/null 2>&1; then sleep 2 - # Check internal endpoint - _INT_READY=$(curl -s -m 5 "http://127.0.0.1:$INACTIVE_PORT/ready" 2>/dev/null || echo "") + # Check internal endpoint (container DNS) + _INT_READY=$(curl -s -m 5 "http://$INACTIVE_NAME:$APP_PORT/ready" 2>/dev/null || echo "") _INT_READY_OK=false if echo "$_INT_READY" | grep -q '"status":"ready"' 2>/dev/null; then _INT_READY_OK=true - _ft_log "msg='truth check: internal endpoint ready' url=http://127.0.0.1:$INACTIVE_PORT/ready" + _ft_log "msg='truth check: internal endpoint ready' url=http://$INACTIVE_NAME:$APP_PORT/ready" else - _ft_log "level=WARN msg='truth check: internal endpoint not ready' url=http://127.0.0.1:$INACTIVE_PORT/ready response=${_INT_READY:0:100}" + _ft_log "level=WARN msg='truth check: internal endpoint not ready' url=http://$INACTIVE_NAME:$APP_PORT/ready response=${_INT_READY:0:100}" fi # Check external endpoint (DNS/Cloudflare/TLS) with latency measurement (SLO monitoring) @@ -921,9 +1010,9 @@ if [ "$_FT_TRUTH_CHECK_PASSED" != "true" ]; then fi # Persist last-known-good snapshot for fast recovery triage (atomic write) -_ft_log "msg='recording last-known-good state' slot=$INACTIVE port=$INACTIVE_PORT" +_ft_log "msg='recording last-known-good state' slot=$INACTIVE container=$INACTIVE_NAME" _SNAP_TMP=$(mktemp "${SNAP_DIR}/last-good.XXXXXX") -printf 'slot=%s port=%s ts=%s\n' "$INACTIVE" "$INACTIVE_PORT" "$(date -Iseconds)" > "$_SNAP_TMP" +printf 'slot=%s container=%s ts=%s\n' "$INACTIVE" "$INACTIVE_NAME" "$(date -Iseconds)" > "$_SNAP_TMP" mv "$_SNAP_TMP" "$LAST_GOOD_FILE" _ft_log "msg='last-known-good snapshot recorded (atomic)' file=$LAST_GOOD_FILE" @@ -963,3 +1052,5 @@ else echo "$MONITORING_HASH" > "$MONITORING_HASH_FILE" _ft_log "msg='monitoring stack restarted'" fi + +_ft_exit 0 "DEPLOY_SUCCESS" "sha=$IMAGE_SHA container=$INACTIVE_NAME slot=$INACTIVE" diff --git a/scripts/rollback.sh b/scripts/rollback.sh index 90218e2..2f2ef11 100644 --- a/scripts/rollback.sh +++ b/scripts/rollback.sh @@ -94,13 +94,13 @@ if ! "$SCRIPT_DIR/deploy-bluegreen.sh" "$PREVIOUS_SHA"; then echo " Active containers:" docker ps --format ' {{.Names}} → {{.Status}} ({{.Ports}})' 2>/dev/null || echo " (docker ps failed)" echo " Active slot file: $(cat "/var/run/api/active-slot" 2>/dev/null || echo 'MISSING')" - echo " Nginx config test: $(sudo nginx -t 2>&1)" + echo " Nginx config test: $(docker exec nginx nginx -t 2>&1)" echo "" echo "Target SHA: $PREVIOUS_SHA" echo "" echo "Action required:" echo " 1. Check container status: docker ps -a" - echo " 2. Check nginx config: sudo nginx -t" + echo " 2. Check nginx config: docker exec nginx nginx -t" echo " 3. Review logs: docker logs api-blue api-green" echo " 4. Manually restore last known good state" echo "=========================================" diff --git a/scripts/vps-setup.sh b/scripts/vps-setup.sh index 679e481..091121f 100644 --- a/scripts/vps-setup.sh +++ b/scripts/vps-setup.sh @@ -32,7 +32,8 @@ REPO_DIR="$DEPLOY_ROOT" LEGACY_REPO_DIR="${DEPLOY_HOME}/FieldTrack-2.0" AUTO_CLEAN_LEGACY_REPO="${AUTO_CLEAN_LEGACY_REPO:-false}" NETWORK="api_network" -NGINX_SITE_LINK="/etc/nginx/conf.d/api.conf" +NGINX_LIVE_DIR="$DEPLOY_ROOT/infra/nginx/live" +NGINX_SITE_LINK="$NGINX_LIVE_DIR/api.conf" # ── Colour output ───────────────────────────────────────────────────────────── GREEN='\033[0;32m' @@ -46,8 +47,10 @@ err() { echo -e "${RED}[✗]${NC} $1"; exit 1; } render_nginx_ssl_config() { local target_file="$1" + mkdir -p "$(dirname "$target_file")" cp "$REPO_DIR/infra/nginx/api.conf" "$target_file" sed -i "s|__API_HOSTNAME__|$DOMAIN|g" "$target_file" + sed -i "s|__ACTIVE_CONTAINER__|api-blue|g" "$target_file" # bootstrap slot } echo "" @@ -332,8 +335,9 @@ log "SSL certificate provisioned for $DOMAIN" # Stage 2: install SSL Nginx config after certs exist log "Phase 11b: Activating SSL Nginx config..." render_nginx_ssl_config "$NGINX_SITE_LINK" +# Validate config with system nginx while it's still running nginx -t && systemctl reload nginx -log "SSL Nginx config activated at $NGINX_SITE_LINK" +log "SSL Nginx config rendered at $NGINX_SITE_LINK" # ============================================================================ # PHASE 12: GHCR Login @@ -385,14 +389,22 @@ else fi # ============================================================================ -# PHASE 14: Start Monitoring Stack +# PHASE 14: Start Monitoring Stack (including Docker nginx) # ============================================================================ log "Phase 14: Starting monitoring stack..." +# Stop system nginx — Docker nginx in the monitoring stack takes over ports 80/443. +# System nginx is no longer needed after cert acquisition; Docker nginx handles +# ACME challenge renewal via /var/www/certbot mount. +log "Phase 14a: Stopping system nginx (Docker nginx takes over)..." +systemctl stop nginx || true +systemctl disable nginx || true +log "System nginx stopped and disabled." + cd "$REPO_DIR/infra" sudo -u "$DEPLOY_USER" docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml up -d -log "Monitoring stack started (Prometheus, Grafana, Node Exporter)" +log "Monitoring stack started (Prometheus, Grafana, Node Exporter, Nginx)" # ============================================================================ # PHASE 15: First Deployment @@ -411,12 +423,11 @@ else sudo -u "$DEPLOY_USER" docker run -d \ --name api-blue \ --network "$NETWORK" \ - -p "127.0.0.1:3001:3000" \ --restart unless-stopped \ --env-file "$ENV_FILE" \ ghcr.io/fieldtrack-tech/api:latest - log "Backend container (api-blue) started on 127.0.0.1:3001." + log "Backend container (api-blue) started (network: $NETWORK)." fi # ============================================================================ @@ -443,7 +454,7 @@ echo "=============================================" echo "" echo " Next steps:" echo " 1. Edit $ENV_FILE with production values" -echo " 2. Verify: curl http://127.0.0.1:3001/health" +echo " 2. Verify: curl https://$DOMAIN/health" echo " 3. Verify: curl https://$DOMAIN/health" echo " 4. Edit $MONITORING_ENV_FILE (set GRAFANA_ADMIN_PASSWORD and METRICS_SCRAPE_TOKEN)" echo " 5. Grafana: https://$DOMAIN/monitor (admin / configured password)" From 04c43581ee4c58e81671ac61788768837f99e739 Mon Sep 17 00:00:00 2001 From: rajashish147 Date: Fri, 3 Apr 2026 17:18:12 +0530 Subject: [PATCH 2/2] feat(nginx): update api_backend configuration to use variable-based resolution for Docker --- infra/nginx/api.conf | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/infra/nginx/api.conf b/infra/nginx/api.conf index 6a13eef..7ba06e3 100644 --- a/infra/nginx/api.conf +++ b/infra/nginx/api.conf @@ -7,10 +7,11 @@ map $http_upgrade $connection_upgrade { '' close; } -upstream api_backend { - server __ACTIVE_CONTAINER__:3000 max_fails=3 fail_timeout=30s; - keepalive 32; -} +# NOTE: No upstream block for api_backend. +# upstream blocks resolve server hostnames at config-load time, which fails +# for Docker service names (api-blue / api-green) that may not exist yet. +# Instead, use a variable + proxy_pass to defer resolution to request time via +# the resolver 127.0.0.11 directive defined in the server block below. limit_req_zone $binary_remote_addr zone=api_rate:10m rate=60r/s; limit_req_zone $binary_remote_addr zone=api_health:10m rate=5r/s; @@ -116,6 +117,10 @@ server { resolver 127.0.0.11 valid=10s; resolver_timeout 5s; + # Variable-based backend URL — resolved at request time via Docker DNS (127.0.0.11). + # __ACTIVE_CONTAINER__ is substituted with api-blue or api-green by deploy script. + set $api_backend "http://__ACTIVE_CONTAINER__:3000"; + # safer host validation (still simple) if ($host !~* ^(__API_HOSTNAME__|localhost|127\.0\.0\.1)$) { return 444; @@ -171,7 +176,7 @@ server { # regressions — nginx won't silently change the upstream path. location = /health { limit_req zone=api_health burst=10 nodelay; - proxy_pass http://api_backend/health; + proxy_pass $api_backend$request_uri; proxy_buffering off; proxy_set_header Host __API_HOSTNAME__; proxy_set_header X-Forwarded-Host $host; @@ -189,7 +194,7 @@ server { allow ::1; deny all; limit_req zone=api_health burst=10 nodelay; - proxy_pass http://api_backend/ready; + proxy_pass $api_backend$request_uri; proxy_buffering off; proxy_set_header Host __API_HOSTNAME__; proxy_set_header X-Forwarded-Host $host; @@ -203,7 +208,7 @@ server { # SSE — open to all origins; application enforces JWT auth location = /admin/events { limit_req zone=api_rate burst=10 nodelay; - proxy_pass http://api_backend; + proxy_pass $api_backend$request_uri; proxy_http_version 1.1; proxy_set_header Connection ''; proxy_set_header Host __API_HOSTNAME__; @@ -218,7 +223,7 @@ server { # MAIN API — open to all origins; application enforces JWT + RBAC location / { limit_req zone=api_rate burst=50 nodelay; - proxy_pass http://api_backend; + proxy_pass $api_backend$request_uri; proxy_http_version 1.1; proxy_set_header Host __API_HOSTNAME__; proxy_set_header X-Forwarded-Host $host;