diff --git a/envs/gcp/prod/main.tf b/envs/gcp/prod/main.tf index 1fdaaf2..6c098cb 100755 --- a/envs/gcp/prod/main.tf +++ b/envs/gcp/prod/main.tf @@ -19,7 +19,7 @@ locals { } - +/* # Network module module "network" { count = local.enable_private_networking ? 1 : 0 @@ -293,8 +293,10 @@ module "lb_backends" { } lb_domain = var.base_domain } +*/ +/* output "gcs_bucket_name" { value = module.gcs.bucket_name } @@ -332,11 +334,6 @@ output "campsite_cloud_run_url" { value = module.campsite_cloud_run.url } -output "project_id" { - description = "GCP project ID" - value = var.project_id -} - output "monitoring_logging_api_enabled" { description = "Whether Logging/Monitoring APIs are enabled" value = module.monitoring.logging_api_enabled && module.monitoring.monitoring_api_enabled @@ -346,3 +343,35 @@ output "lb_ip" { description = "The public Anycast IP address of the load balancer" value = var.enable_lb ? module.lb_backends[0].lb_ip : null } +*/ + +output "project_id" { + description = "GCP project ID" + value = var.project_id +} + +resource "google_compute_instance" "orion_client_vm" { + name = "${var.app_name}-orion-client-vm" + machine_type = "e2-standard-4" + zone = var.zone + + boot_disk { + initialize_params { + image = "debian-cloud/debian-13" + size = 80 + } + } + + network_interface { + network = "buck2hub-vpc3" + subnetwork = "buck2hub-subnet" + + access_config {} + } + + metadata_startup_script = file("${path.module}/scripts/startup-orion-client.sh") + + service_account { + scopes = ["cloud-platform"] + } +} diff --git a/envs/gcp/prod/scripts/startup-orion-client.sh b/envs/gcp/prod/scripts/startup-orion-client.sh new file mode 100755 index 0000000..956847f --- /dev/null +++ b/envs/gcp/prod/scripts/startup-orion-client.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec > >(tee -a /var/log/orion-client-startup.log) 2>&1 + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + git-lfs \ + gettext-base \ + netcat-openbsd \ + procps \ + zstd \ + fuse3 \ + libssl3 \ + build-essential \ + pkg-config \ + cmake \ + clang \ + llvm-dev \ + libclang-dev \ + libssl-dev \ + libfuse3-dev \ + protobuf-compiler + +echo "user_allow_other" >> /etc/fuse.conf || true + +mkdir -p \ + /opt/orion-client \ + /opt/orion-client/src \ + /opt/orion-client/bin \ + /opt/orion-client/log \ + /data/scorpio/store \ + /data/scorpio/antares/upper \ + /data/scorpio/antares/cl \ + /data/scorpio/antares/mnt \ + /workspace/mount + +# Install Rust toolchain if missing +if ! command -v rustc >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +fi +export PATH="/root/.cargo/bin:${PATH}" + +# Install Buck2 +BUCK2_VERSION="2025-06-01" +ARCH="x86_64-unknown-linux-musl" +curl -fsSL -o /usr/local/bin/buck2.zst "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/buck2-${ARCH}.zst" +zstd -d /usr/local/bin/buck2.zst -o /usr/local/bin/buck2 +chmod +x /usr/local/bin/buck2 + +# Create wrapper that behaves like the container entrypoint (start embedded scorpio then orion) +cat >/usr/local/bin/orion-worker-wrapper <<'WRAPPER' +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[orion-worker-wrapper] $*"; } + +: "${SERVER_WS:?SERVER_WS is required}" + +# Defaults aligned with mega/orion/entrypoint.sh +: "${SCORPIO_API_BASE_URL:=http://127.0.0.1:2725}" +: "${SCORPIO_HTTP_ADDR:=0.0.0.0:2725}" +: "${ORION_WORKER_START_SCORPIO:=true}" + +: "${SCORPIO_STORE_PATH:=/data/scorpio/store}" +: "${SCORPIO_WORKSPACE:=/workspace/mount}" + +mkdir -p "$SCORPIO_STORE_PATH" "$SCORPIO_WORKSPACE" \ + /data/scorpio/antares/upper /data/scorpio/antares/cl /data/scorpio/antares/mnt + +if [ "$ORION_WORKER_START_SCORPIO" != "false" ] && [ "$ORION_WORKER_START_SCORPIO" != "0" ]; then + if [ ! -e /dev/fuse ]; then + log "ERROR: /dev/fuse not found; Scorpio requires FUSE" + exit 1 + fi + + log "Starting embedded scorpio..." + scorpio -c /etc/scorpio/scorpio.toml --http-addr "$SCORPIO_HTTP_ADDR" & + scorpio_pid=$! + + # Wait for scorpio to listen + port="${SCORPIO_HTTP_ADDR##*:}" + for i in $(seq 1 60); do + if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + if ! nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then + log "ERROR: scorpio did not become ready on 127.0.0.1:$port" + kill "$scorpio_pid" 2>/dev/null || true + exit 1 + fi + + log "Scorpio ready on 127.0.0.1:$port" + export SCORPIO_API_BASE_URL="http://127.0.0.1:$port" +fi + +log "Starting orion worker..." +exec orion +WRAPPER + +chmod +x /usr/local/bin/orion-worker-wrapper + +# Write and enable systemd service for orion worker +cat >/etc/systemd/system/orion-worker.service <<'EOF' +[Unit] +Description=Orion Worker Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=root +Environment="SERVER_WS=wss://buck2hub-orion-504513835593.asia-east1.run.app/ws" +Environment="SCORPIO_BASE_URL=https://buck2hub-mono-504513835593.asia-east1.run.app" +Environment="SCORPIO_LFS_URL=https://buck2hub-mono-504513835593.asia-east1.run.app" +Environment="SCORPIO_STORE_PATH=/data/scorpio/store" +Environment="SCORPIO_WORKSPACE=/workspace/mount" +Environment="ORION_WORKER_START_SCORPIO=true" +Environment="SCORPIO_HTTP_ADDR=0.0.0.0:2725" +ExecStart=/usr/local/bin/orion-worker-wrapper +Restart=always +RestartSec=5 +LimitNOFILE=1048576 + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable orion-worker.service + diff --git a/envs/gcp/prod/terraform.tfvars.example b/envs/gcp/prod/terraform.tfvars.example index 94d28ba..d272e4a 100755 --- a/envs/gcp/prod/terraform.tfvars.example +++ b/envs/gcp/prod/terraform.tfvars.example @@ -8,12 +8,11 @@ base_domain = "buck2hub.com" # used for certificates / app config # Region / zone (optional overrides) region = "asia-east1" -# zone = "us-central1-b" +zone = "asia-east1-a" # GCS (object storage) gcs_force_destroy = false - # Cloud Run Images # Note: GitHub Actions will push images directly to ${app_name}-repo. app_image = "us-central1-docker.pkg.dev/infra-20250121-20260121-0235/mega/mega:mono-0.1.0-pre-release-amd64" @@ -21,7 +20,6 @@ ui_image = "us-central1-docker.pkg.dev/infra-20250121-20260121-0235/mega/m orion_image = "us-central1-docker.pkg.dev/infra-20250121-20260121-0235/mega/mega:orion-server-0.1.0-pre-release-amd64" campsite_image = "us-central1-docker.pkg.dev/infra-20250121-20260121-0235/mega/mega:campsite-0.1.0-pre-release-amd64" - # Database (Cloud SQL) cloud_sql_pg_name = "buck2hub" cloud_sql_mysql_name = "campsite_api_buck2hub" @@ -29,7 +27,7 @@ db_username = "gitmega" db_password = "your-prod-db-password" # Testing (option 1): enable public IP while keeping private networking to avoid empty network references -cloud_sql_enable_public_ip = true +cloud_sql_enable_public_ip = true # VPC Connector CIDR (Required for the module to avoid 400 error) vpc_connector_cidr = "10.8.0.0/28" diff --git a/envs/gcp/prod/variables.tf b/envs/gcp/prod/variables.tf index 469088a..d89baf7 100755 --- a/envs/gcp/prod/variables.tf +++ b/envs/gcp/prod/variables.tf @@ -10,6 +10,22 @@ variable "app_name" { default = "mega" } +variable "orion_client_server_ws" { + type = string + description = "Orion Server WebSocket URL for orion worker (e.g. wss://orion.example.com/ws or ws://10.x.x.x:8004/ws)." +} + +variable "orion_client_scorpio_base_url" { + type = string + description = "Mono base URL for scorpio (e.g. https://git.example.com or http://10.x.x.x:8000)." +} + +variable "orion_client_src_tgz_path" { + type = string + description = "Path on VM to the uploaded orion client source tarball (.tgz). This file should be uploaded manually from Cloud Shell." + default = "/opt/orion-client/src.tgz" +} + variable "region" { type = string default = "us-central1" @@ -18,7 +34,7 @@ variable "region" { variable "zone" { type = string description = "GCP zone for zonal resources (e.g. Filestore)." - default = "" + default = "us-central1-a" } variable "base_domain" { @@ -27,9 +43,6 @@ variable "base_domain" { default = "buck2hub.com" } - - - variable "enable_lb" { type = bool description = "Whether to enable Global HTTPS Load Balancer"