deploy.sh

#!/usr/bin/env bash
# deploy.sh — direct host deployment for backend + frontend (no Docker)
set -Eeuo pipefail

export DEBIAN_FRONTEND=noninteractive
export NEEDRESTART_MODE=a

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_DIR=""
BACKEND_SERVICE="layout-backend"
FRONTEND_SERVICE="layout-frontend"
NGINX_CONF="/etc/nginx/sites-available/layout-host"
BACKEND_PORT="8000"
FRONTEND_PORT="3000"
BACKEND_MAX_POOL_WORKERS="${BACKEND_MAX_POOL_WORKERS:-1}"
MODEL_SHARE_BASE_URL="https://owncloud.gwdg.de/remote.php/dav/public-files/PyQ2nN6aKpypKfG/Apps/Layout%20App/model%20weights"
TMP_ROOT=""
PUBLIC_IP=""
APP_TITLE="Manuscript Layout Analysis"

cleanup() {
  [ -n "${TMP_ROOT:-}" ] && [ -d "${TMP_ROOT:-}" ] && rm -rf "$TMP_ROOT" || true
}
trap cleanup EXIT

log() { echo "==> $*"; }
fail() { echo; echo "!! $*"; exit 1; }

dump_backend_logs() {
  echo
  echo "-- ${BACKEND_SERVICE} journal (last 80 lines) --"
  sudo journalctl -u "${BACKEND_SERVICE}" -n 80 --no-pager || true
}

detect_repo_dir() {
  if [ -d "$SCRIPT_DIR/.git" ] && [ -d "$SCRIPT_DIR/backend" ] && [ -d "$SCRIPT_DIR/frontend" ]; then
    echo "$SCRIPT_DIR"; return
  fi
  if [ -d "$PWD/.git" ] && [ -d "$PWD/backend" ] && [ -d "$PWD/frontend" ]; then
    echo "$PWD"; return
  fi
  if [ -d "$PWD/layout/.git" ] && [ -d "$PWD/layout/backend" ] && [ -d "$PWD/layout/frontend" ]; then
    echo "$PWD/layout"; return
  fi
  if [ -d "$PWD/Mergen/.git" ] && [ -d "$PWD/Mergen/backend" ] && [ -d "$PWD/Mergen/frontend" ]; then
    echo "$PWD/Mergen"; return
  fi
  echo ""
}

detect_public_ip() {
  local ip=""
  ip="$(curl -4fsS https://api.ipify.org 2>/dev/null || true)"
  [ -n "$ip" ] || ip="$(curl -4fsS https://ifconfig.me 2>/dev/null || true)"
  [ -n "$ip" ] || ip="$(curl -4fsS https://icanhazip.com 2>/dev/null | tr -d '[:space:]' || true)"
  echo "$ip"
}

ensure_node_20() {
  local major="0"
  if command -v node >/dev/null 2>&1; then
    major="$(node -v | sed 's/^v//' | cut -d. -f1)"
  fi
  if [ "$major" -lt 20 ]; then
    log "Installing Node.js 20..."
    curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
    sudo apt-get install -y --no-install-recommends nodejs
  fi
}

check_weights() {
  [ -f "$REPO_DIR/backend/models/best_catmus.pt" ] &&
  [ -f "$REPO_DIR/backend/models/best_emanuskript_segmentation.pt" ] &&
  [ -f "$REPO_DIR/backend/models/best_zone_detection.pt" ]
}

download_model_weight() {
  local filename="$1"
  local target_dir="$2"
  local tmp_file="${target_dir}/${filename}.tmp"
  local final_file="${target_dir}/${filename}"

  curl -fL --retry 3 --retry-delay 2 \
    "${MODEL_SHARE_BASE_URL}/${filename}" \
    -o "${tmp_file}"
  mv "${tmp_file}" "${final_file}"
}

download_all_model_weights() {
  local target_dir="$1"
  mkdir -p "${target_dir}"

  log "Downloading model weights from OwnCloud share..."
  download_model_weight "best_catmus.pt" "${target_dir}"
  download_model_weight "best_emanuskript_segmentation.pt" "${target_dir}"
  download_model_weight "best_zone_detection.pt" "${target_dir}"
}

ensure_model_weights() {
  local target_dir="${REPO_DIR}/backend/models"
  mkdir -p "${target_dir}"

  if ! check_weights; then
    download_all_model_weights "${target_dir}"
  fi
}

free_space() {
  log "Freeing disk space..."
  sudo apt-get clean || true
  sudo apt-get autoremove -y || true
  sudo rm -rf /tmp/* /var/tmp/* || true
  rm -rf "$HOME/.cache" "$HOME/.npm" || true
  rm -rf "$REPO_DIR/frontend/.next" "$REPO_DIR/frontend/node_modules" || true
  rm -rf "$REPO_DIR/backend/.venv" "$REPO_DIR/.deploy-tmp" || true
  sudo journalctl --vacuum-time=1d || true
  if command -v docker >/dev/null 2>&1; then
    sudo docker system prune -a -f --volumes || true
  fi
}

require_space() {
  local avail_kb
  avail_kb="$(df --output=avail / | tail -1 | tr -d ' ')"
  [ -n "$avail_kb" ] || fail "Could not determine free disk space."
  [ "$avail_kb" -ge 1500000 ] || fail "Need at least ~1.5 GB free on / before deployment."
}

disable_conflicting_services() {
  log "Disabling conflicting web services..."
  for svc in caddy apache2; do
    if systemctl list-unit-files 2>/dev/null | grep -q "^${svc}\.service"; then
      sudo systemctl stop "$svc" || true
      sudo systemctl disable "$svc" || true
    fi
  done
}

disable_old_public_configs() {
  log "Disabling old public nginx configs..."
  sudo mkdir -p /etc/nginx/disabled-by-layout
  sudo rm -f /etc/nginx/sites-enabled/* || true

  while IFS= read -r -d '' f; do
    if sudo grep -qiE 'duckdns\.org|thelayout|layout\.duckdns|thelayout\.duckdns' "$f"; then
      sudo mv "$f" "/etc/nginx/disabled-by-layout/$(basename "$f").bak.$(date +%s)" || true
    fi
  done < <(sudo find /etc/nginx \
    \( -path "/etc/nginx/sites-available/*" -o -path "/etc/nginx/conf.d/*" \) \
    \( -type f -o -type l \) -print0 2>/dev/null)
}

write_backend_service() {
  sudo tee "/etc/systemd/system/${BACKEND_SERVICE}.service" >/dev/null <<EOF
[Unit]
Description=Layout Backend API
After=network.target

[Service]
Type=simple
User=${USER}
WorkingDirectory=${REPO_DIR}/backend
Environment=MODEL_DIR=${REPO_DIR}/backend/models
Environment=CORS_ORIGINS=http://${PUBLIC_IP}
Environment=MAX_POOL_WORKERS=${BACKEND_MAX_POOL_WORKERS}
Environment=OMP_NUM_THREADS=1
Environment=OPENBLAS_NUM_THREADS=1
Environment=MKL_NUM_THREADS=1
Environment=NUMEXPR_NUM_THREADS=1
Environment=PYTHONUNBUFFERED=1
ExecStart=${REPO_DIR}/backend/.venv/bin/uvicorn app.main:app --host 127.0.0.1 --port ${BACKEND_PORT} --workers 1
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
}

write_frontend_service() {
  sudo tee "/etc/systemd/system/${FRONTEND_SERVICE}.service" >/dev/null <<EOF
[Unit]
Description=Layout Frontend
After=network.target ${BACKEND_SERVICE}.service
Requires=${BACKEND_SERVICE}.service

[Service]
Type=simple
User=${USER}
WorkingDirectory=${REPO_DIR}/frontend/.next/standalone
Environment=NODE_ENV=production
Environment=PORT=${FRONTEND_PORT}
Environment=HOSTNAME=127.0.0.1
Environment=INTERNAL_BACKEND_URL=http://127.0.0.1:${BACKEND_PORT}
ExecStart=/usr/bin/env node server.js
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
}

write_nginx_config() {
  sudo tee "$NGINX_CONF" >/dev/null <<EOF
server {
    listen 80 default_server;
    listen [::]:80 default_server;
    server_name _;

    client_max_body_size 200M;

    location = /api {
        return 301 /api/;
    }

    location ^~ /api/ {
        proxy_pass http://127.0.0.1:${BACKEND_PORT}/api/;
        proxy_http_version 1.1;
        proxy_request_buffering off;
        proxy_buffering off;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_read_timeout 300s;
        proxy_send_timeout 300s;
    }

    location = /docs {
        proxy_pass http://127.0.0.1:${BACKEND_PORT}/docs;
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
    }

    location = /openapi.json {
        proxy_pass http://127.0.0.1:${BACKEND_PORT}/openapi.json;
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
    }

    location = /redoc {
        proxy_pass http://127.0.0.1:${BACKEND_PORT}/redoc;
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
    }

    location / {
        proxy_pass http://127.0.0.1:${FRONTEND_PORT};
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_set_header Upgrade \$http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_read_timeout 300s;
        proxy_send_timeout 300s;
    }
}
EOF
}

verify_local_frontend() {
  curl -fsS "http://127.0.0.1:${FRONTEND_PORT}" | grep -q "${APP_TITLE}" \
    || fail "Local frontend on port ${FRONTEND_PORT} is not serving the expected app."
}

verify_local_nginx() {
  curl -fsS "http://127.0.0.1" | grep -q "${APP_TITLE}" \
    || fail "Local nginx is not serving the expected frontend."
}

verify_api() {
  curl -fsS "http://127.0.0.1/api/health" >/dev/null \
    || fail "nginx -> backend health route is broken."
  curl -fsS "http://127.0.0.1/api/classes" >/dev/null \
    || fail "nginx -> backend classes route is broken."
}

run_prediction_request() {
  local url="$1"
  local response_file="$2"

  curl -sS -o "${response_file}" -w "%{http_code}" -X POST \
    -F "image=@${smoke_image}" \
    -F "confidence=0.25" \
    -F "iou=0.3" \
    "${url}"
}

verify_prediction_flow() {
  local smoke_image response response_file backend_response_file http_code backend_http_code task_id annotated_path
  smoke_image="${TMP_ROOT}/smoke-test.jpg"
  response_file="${TMP_ROOT}/predict-response.json"
  backend_response_file="${TMP_ROOT}/predict-response-backend.json"

  "$REPO_DIR/backend/.venv/bin/python" - "$smoke_image" <<'PY'
from PIL import Image
import sys

Image.new("RGB", (512, 512), "white").save(sys.argv[1])
PY

  http_code="$(run_prediction_request "http://127.0.0.1/api/predict/single" "${response_file}")" || {
    dump_backend_logs
    fail "nginx -> backend single prediction route is broken."
  }

  if [ "${http_code}" != "200" ]; then
    echo
    echo "-- nginx predict response body --"
    cat "${response_file}" || true

    backend_http_code="$(run_prediction_request "http://127.0.0.1:${BACKEND_PORT}/api/predict/single" "${backend_response_file}")" || true
    if [ -n "${backend_http_code:-}" ]; then
      echo
      echo "-- backend direct predict response (${backend_http_code}) --"
      cat "${backend_response_file}" || true
    fi

    dump_backend_logs
    fail "nginx -> backend single prediction route returned HTTP ${http_code}."
  fi

  response="$(cat "${response_file}")"

  task_id="$(
    printf '%s' "$response" | "$REPO_DIR/backend/.venv/bin/python" -c \
      'import json,sys; print(json.load(sys.stdin)["task_id"])'
  )"

  annotated_path="$(
    printf '%s' "$response" | "$REPO_DIR/backend/.venv/bin/python" -c \
      'import json,sys; print(json.load(sys.stdin)["annotated_image_url"])'
  )"

  [ -n "$task_id" ] || fail "Prediction response did not include a task_id."
  [ -n "$annotated_path" ] || fail "Prediction response did not include annotated_image_url."

  curl -fsS "http://127.0.0.1${annotated_path}" -o /dev/null -D - | grep -qi '^content-type: image/jpeg' \
    || fail "Predicted annotated image is not reachable through nginx."
}

verify_public_route() {
  if curl -fsS "http://${PUBLIC_IP}" | grep -q "${APP_TITLE}"; then
    log "Public IP is serving the expected frontend."
  else
    echo
    echo "!! Local deployment is correct, but public IP is not serving the same frontend."
  fi
}

log "Installing base packages..."
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends \
  ca-certificates curl git nginx python3 python3-pip python3-venv \
  build-essential libgl1 libglib2.0-0 libsm6 libxext6 libxrender1 \
  libspatialindex-dev

log "Resolving repository path..."
REPO_DIR="$(detect_repo_dir)"
if [ -z "$REPO_DIR" ]; then
  git clone https://github.com/emanuskript/Mergen.git "$PWD/Mergen"
  REPO_DIR="$PWD/Mergen"
fi
cd "$REPO_DIR"

ensure_node_20

log "Ensuring model weights are present..."
ensure_model_weights

PUBLIC_IP="$(detect_public_ip)"
[ -n "$PUBLIC_IP" ] || fail "Unable to detect public IP."

free_space
require_space
disable_conflicting_services
disable_old_public_configs

TMP_ROOT="${REPO_DIR}/.deploy-tmp"
mkdir -p "$TMP_ROOT"
export TMPDIR="$TMP_ROOT"
export TEMP="$TMP_ROOT"
export TMP="$TMP_ROOT"

log "Preparing backend venv..."
python3 -m venv "$REPO_DIR/backend/.venv"
"$REPO_DIR/backend/.venv/bin/pip" install --upgrade pip setuptools wheel

log "Installing backend dependencies..."
TMPDIR="$TMP_ROOT" "$REPO_DIR/backend/.venv/bin/pip" install --no-cache-dir \
  --index-url https://download.pytorch.org/whl/cpu torch torchvision
TMPDIR="$TMP_ROOT" "$REPO_DIR/backend/.venv/bin/pip" install --no-cache-dir \
  -e "$REPO_DIR/backend"

log "Validating backend model checkpoints..."
MODEL_DIR="${REPO_DIR}/backend/models" \
  "$REPO_DIR/backend/.venv/bin/python" -m app.scripts.validate_models \
  || {
    log "Model validation failed; re-downloading all weights from OwnCloud share..."
    download_all_model_weights "${REPO_DIR}/backend/models"
    MODEL_DIR="${REPO_DIR}/backend/models" \
      "$REPO_DIR/backend/.venv/bin/python" -m app.scripts.validate_models \
      || fail "One or more model checkpoints are invalid even after re-download."
  }

log "Building frontend..."
cd "$REPO_DIR/frontend"
rm -rf .next node_modules
if [ -f package-lock.json ]; then
  TMPDIR="$TMP_ROOT" npm ci --cache "$TMP_ROOT/.npm-cache"
else
  TMPDIR="$TMP_ROOT" npm install --cache "$TMP_ROOT/.npm-cache"
fi
INTERNAL_BACKEND_URL="http://127.0.0.1:${BACKEND_PORT}" TMPDIR="$TMP_ROOT" npm run build
mkdir -p .next/standalone/.next
rm -rf .next/standalone/.next/static .next/standalone/public
cp -R .next/static .next/standalone/.next/static
cp -R public .next/standalone/public
cd "$REPO_DIR"

log "Writing services and nginx config..."
write_backend_service
write_frontend_service
write_nginx_config
sudo ln -sf "$NGINX_CONF" /etc/nginx/sites-enabled/layout-host

log "Restarting services..."
sudo systemctl daemon-reload
sudo systemctl enable "$BACKEND_SERVICE" "$FRONTEND_SERVICE" nginx
sudo systemctl stop "$FRONTEND_SERVICE" || true
sudo pkill -f "next.*3000" || true
sudo pkill -f "next-server" || true
sudo pkill -f "next start" || true
sudo systemctl restart "$BACKEND_SERVICE"
sudo systemctl start "$FRONTEND_SERVICE"
sudo nginx -t
sudo systemctl restart nginx

sleep 3

sudo systemctl is-active --quiet "$BACKEND_SERVICE" || fail "Backend service not active."
sudo systemctl is-active --quiet "$FRONTEND_SERVICE" || fail "Frontend service not active."
sudo systemctl is-active --quiet nginx || fail "nginx not active."

verify_local_frontend
verify_local_nginx
verify_api
verify_prediction_flow
verify_public_route

echo
echo "==> Deployment complete"
echo "Frontend URL: http://${PUBLIC_IP}"
echo "Backend local: http://127.0.0.1:${BACKEND_PORT}"
echo "Services: ${BACKEND_SERVICE}, ${FRONTEND_SERVICE}, nginx"