From ac00d7e0a52c1ecef3b3605f43931c58a3b1cec6 Mon Sep 17 00:00:00 2001 From: Hongze Gao <15101764808@163.com> Date: Sat, 14 Feb 2026 21:57:49 +0800 Subject: [PATCH 1/3] gcp: add VM-only native Orion client bootstrap for prod env - Add a GCE VM (e2-standard-4) attached to existing VPC/subnet via data sources - Install build/runtime deps, rust toolchain and buck2 in startup script - Provide install_from_tgz.sh to build orion/scorpio from uploaded source tarball - Disable non-VM modules/outputs via block comments for local testing Signed-off-by: Hongze Gao <15101764808@163.com> --- envs/gcp/prod/main.tf | 103 ++++++++++++++++++++++++++++++++++--- envs/gcp/prod/variables.tf | 21 ++++++-- 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/envs/gcp/prod/main.tf b/envs/gcp/prod/main.tf index 1fdaaf2..c88e686 100755 --- a/envs/gcp/prod/main.tf +++ b/envs/gcp/prod/main.tf @@ -19,7 +19,7 @@ locals { } - +/* # Network module module "network" { count = local.enable_private_networking ? 1 : 0 @@ -293,8 +293,10 @@ module "lb_backends" { } lb_domain = var.base_domain } +*/ +/* output "gcs_bucket_name" { value = module.gcs.bucket_name } @@ -332,11 +334,6 @@ output "campsite_cloud_run_url" { value = module.campsite_cloud_run.url } -output "project_id" { - description = "GCP project ID" - value = var.project_id -} - output "monitoring_logging_api_enabled" { description = "Whether Logging/Monitoring APIs are enabled" value = module.monitoring.logging_api_enabled && module.monitoring.monitoring_api_enabled @@ -346,3 +343,97 @@ output "lb_ip" { description = "The public Anycast IP address of the load balancer" value = var.enable_lb ? module.lb_backends[0].lb_ip : null } +*/ + +output "project_id" { + description = "GCP project ID" + value = var.project_id +} + +data "google_compute_network" "existing" { + name = local.network_name +} + +data "google_compute_subnetwork" "existing" { + name = local.subnet_name + region = var.region +} + +resource "google_compute_instance" "orion_client_vm" { + name = "${var.app_name}-orion-client-vm" + machine_type = "e2-standard-4" + zone = var.zone + + boot_disk { + initialize_params { + image = "debian-cloud/debian-12" + size = 80 + } + } + + network_interface { + network = data.google_compute_network.existing.self_link + subnetwork = data.google_compute_subnetwork.existing.self_link + + access_config { + } + } + + metadata_startup_script = <<-EOT + #!/usr/bin/env bash + set -euo pipefail + + exec > >(tee -a /var/log/orion-client-startup.log) 2>&1 + + export DEBIAN_FRONTEND=noninteractive + + apt-get update + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + git-lfs \ + gettext-base \ + netcat-openbsd \ + procps \ + zstd \ + fuse3 \ + libfuse3-3 \ + libssl3 \ + build-essential \ + pkg-config \ + cmake \ + clang \ + llvm-dev \ + libclang-dev \ + libssl-dev \ + libfuse3-dev \ + protobuf-compiler + + echo "user_allow_other" >> /etc/fuse.conf || true + + mkdir -p \ + /opt/orion-client \ + /opt/orion-client/src \ + /data/scorpio/store \ + /data/scorpio/antares/upper \ + /data/scorpio/antares/cl \ + /data/scorpio/antares/mnt \ + /workspace/mount + + if ! command -v rustc >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + fi + export PATH="/root/.cargo/bin:${PATH}" + + BUCK2_VERSION="2025-06-01" + ARCH="x86_64-unknown-linux-musl" + curl -fsSL -o /usr/local/bin/buck2.zst "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/buck2-${ARCH}.zst" + zstd -d /usr/local/bin/buck2.zst -o /usr/local/bin/buck2 + chmod +x /usr/local/bin/buck2 + + cat >/opt/orion-client/install_from_tgz.sh <<'SCRIPT' + #!/usr/bin/env bash + set -euo pipefail + + TGZ_PATH="${1:-/opt/orion-client/src.tgz}" \ No newline at end of file diff --git a/envs/gcp/prod/variables.tf b/envs/gcp/prod/variables.tf index 469088a..d89baf7 100755 --- a/envs/gcp/prod/variables.tf +++ b/envs/gcp/prod/variables.tf @@ -10,6 +10,22 @@ variable "app_name" { default = "mega" } +variable "orion_client_server_ws" { + type = string + description = "Orion Server WebSocket URL for orion worker (e.g. wss://orion.example.com/ws or ws://10.x.x.x:8004/ws)." +} + +variable "orion_client_scorpio_base_url" { + type = string + description = "Mono base URL for scorpio (e.g. https://git.example.com or http://10.x.x.x:8000)." +} + +variable "orion_client_src_tgz_path" { + type = string + description = "Path on VM to the uploaded orion client source tarball (.tgz). This file should be uploaded manually from Cloud Shell." + default = "/opt/orion-client/src.tgz" +} + variable "region" { type = string default = "us-central1" @@ -18,7 +34,7 @@ variable "region" { variable "zone" { type = string description = "GCP zone for zonal resources (e.g. Filestore)." - default = "" + default = "us-central1-a" } variable "base_domain" { @@ -27,9 +43,6 @@ variable "base_domain" { default = "buck2hub.com" } - - - variable "enable_lb" { type = bool description = "Whether to enable Global HTTPS Load Balancer" From 06717e999aa02b8652a7c34897c7ae0717052346 Mon Sep 17 00:00:00 2001 From: Hongze Gao <15101764808@163.com> Date: Sun, 22 Feb 2026 18:17:50 +0800 Subject: [PATCH 2/3] feat(gcp): implement bare-metal VM deployment for Orion Client - Replace Cloud Run with GCE VM for Orion Client execution - Add comprehensive bootstrapping script in metadata_startup_script - Implement automated source extraction and cargo build process on VM - Configure systemd services for Scorpio daemon and Orion Worker - Update network interface to use existing buck2hub-vpc3 and subnet - Add required variables for WSS and Mono service endpoints - Comment out non-essential resources for isolated VM testing Signed-off-by: Hongze Gao <15101764808@163.com> --- envs/gcp/prod/main.tf | 157 ++++++++++++++++++++++--- envs/gcp/prod/terraform.tfvars.example | 6 +- 2 files changed, 143 insertions(+), 20 deletions(-) diff --git a/envs/gcp/prod/main.tf b/envs/gcp/prod/main.tf index c88e686..2231ac5 100755 --- a/envs/gcp/prod/main.tf +++ b/envs/gcp/prod/main.tf @@ -350,15 +350,6 @@ output "project_id" { value = var.project_id } -data "google_compute_network" "existing" { - name = local.network_name -} - -data "google_compute_subnetwork" "existing" { - name = local.subnet_name - region = var.region -} - resource "google_compute_instance" "orion_client_vm" { name = "${var.app_name}-orion-client-vm" machine_type = "e2-standard-4" @@ -372,11 +363,10 @@ resource "google_compute_instance" "orion_client_vm" { } network_interface { - network = data.google_compute_network.existing.self_link - subnetwork = data.google_compute_subnetwork.existing.self_link + network = "buck2hub-vpc3" + subnetwork = "buck2hub-subnet" - access_config { - } + access_config {} } metadata_startup_script = <<-EOT @@ -415,25 +405,160 @@ resource "google_compute_instance" "orion_client_vm" { mkdir -p \ /opt/orion-client \ /opt/orion-client/src \ + /opt/orion-client/bin \ + /opt/orion-client/log \ /data/scorpio/store \ /data/scorpio/antares/upper \ /data/scorpio/antares/cl \ /data/scorpio/antares/mnt \ /workspace/mount + # Install Rust toolchain if missing if ! command -v rustc >/dev/null 2>&1; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y fi - export PATH="/root/.cargo/bin:${PATH}" + export PATH="/root/.cargo/bin:$${PATH}" + # Install Buck2 BUCK2_VERSION="2025-06-01" ARCH="x86_64-unknown-linux-musl" - curl -fsSL -o /usr/local/bin/buck2.zst "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/buck2-${ARCH}.zst" + curl -fsSL -o /usr/local/bin/buck2.zst "https://github.com/facebook/buck2/releases/download/$${BUCK2_VERSION}/buck2-$${ARCH}.zst" zstd -d /usr/local/bin/buck2.zst -o /usr/local/bin/buck2 chmod +x /usr/local/bin/buck2 + # Create wrapper that behaves like the container entrypoint (start embedded scorpio then orion) + cat >/usr/local/bin/orion-worker-wrapper <<'WRAPPER' + #!/usr/bin/env bash + set -euo pipefail + + log() { echo "[orion-worker-wrapper] $*"; } + + : "${SERVER_WS:?SERVER_WS is required}" + + # Defaults aligned with mega/orion/entrypoint.sh + : "${SCORPIO_API_BASE_URL:=http://127.0.0.1:2725}" + : "${SCORPIO_HTTP_ADDR:=0.0.0.0:2725}" + : "${ORION_WORKER_START_SCORPIO:=true}" + + : "${SCORPIO_STORE_PATH:=/data/scorpio/store}" + : "${SCORPIO_WORKSPACE:=/workspace/mount}" + + mkdir -p "$SCORPIO_STORE_PATH" "$SCORPIO_WORKSPACE" \ + /data/scorpio/antares/upper /data/scorpio/antares/cl /data/scorpio/antares/mnt + + if [ "$ORION_WORKER_START_SCORPIO" != "false" ] && [ "$ORION_WORKER_START_SCORPIO" != "0" ]; then + if [ ! -e /dev/fuse ]; then + log "ERROR: /dev/fuse not found; Scorpio requires FUSE" + exit 1 + fi + + log "Starting embedded scorpio..." + # Use the config file prepared during installation + scorpio -c /etc/scorpio/scorpio.toml --http-addr "$SCORPIO_HTTP_ADDR" & + scorpio_pid=$! + + # Wait for scorpio to listen + port="${SCORPIO_HTTP_ADDR##*:}" + for i in $(seq 1 60); do + if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + if ! nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then + log "ERROR: scorpio did not become ready on 127.0.0.1:$port" + kill "$scorpio_pid" 2>/dev/null || true + exit 1 + fi + + log "Scorpio ready on 127.0.0.1:$port" + export SCORPIO_API_BASE_URL="http://127.0.0.1:$port" + fi + + log "Starting orion worker..." + exec orion +WRAPPER + chmod +x /usr/local/bin/orion-worker-wrapper + + # Install/build script: user uploads TGZ to /opt/orion-client/src.tgz, then runs this script. cat >/opt/orion-client/install_from_tgz.sh <<'SCRIPT' #!/usr/bin/env bash set -euo pipefail - TGZ_PATH="${1:-/opt/orion-client/src.tgz}" \ No newline at end of file + TGZ_PATH="${1:-/opt/orion-client/src.tgz}" + SRC_DIR="/opt/orion-client/src" + + if [ ! -f "$TGZ_PATH" ]; then + echo "Error: Tarball not found at $TGZ_PATH" + exit 1 + fi + + rm -rf "$SRC_DIR"/* + + echo "Extracting source..." + # Note: no --strip-components because the tgz root is the workspace root + tar -xzf "$TGZ_PATH" -C "$SRC_DIR" + + echo "Preparing scorpio config..." + mkdir -p /etc/scorpio + if [ -f "$SRC_DIR/scorpio/scorpio.toml" ]; then + cp -f "$SRC_DIR/scorpio/scorpio.toml" /etc/scorpio/scorpio.toml + fi + + echo "Building orion and scorpio (release)..." + cd "$SRC_DIR" + export PATH="/root/.cargo/bin:$PATH" + + # Use a stable target dir to speed up rebuilds across reboots + export CARGO_TARGET_DIR=/opt/orion-client/target + + cargo build --release --package orion --package scorpio + + echo "Installing binaries..." + install -m 0755 "$CARGO_TARGET_DIR/release/orion" /usr/local/bin/orion + install -m 0755 "$CARGO_TARGET_DIR/release/scorpio" /usr/local/bin/scorpio + + echo "Writing systemd unit..." + cat >/etc/systemd/system/orion-worker.service < Date: Wed, 25 Feb 2026 12:24:04 +0800 Subject: [PATCH 3/3] gcp: provision orion client vm with standalone startup script - add google_compute_instance.orion_client_vm to run the orion client on a dedicated GCE VM - switch boot image to debian-cloud/debian-13 and install required runtime deps (rust, buck2, fuse3, etc.) - factor metadata_startup_script into scripts/startup-orion-client.sh for readability and reuse - create orion-worker-wrapper on the VM and register a systemd orion-worker.service unit Signed-off-by: Hongze Gao <15101764808@163.com> --- envs/gcp/prod/main.tf | 191 +----------------- envs/gcp/prod/scripts/startup-orion-client.sh | 139 +++++++++++++ 2 files changed, 141 insertions(+), 189 deletions(-) create mode 100755 envs/gcp/prod/scripts/startup-orion-client.sh diff --git a/envs/gcp/prod/main.tf b/envs/gcp/prod/main.tf index 2231ac5..6c098cb 100755 --- a/envs/gcp/prod/main.tf +++ b/envs/gcp/prod/main.tf @@ -357,7 +357,7 @@ resource "google_compute_instance" "orion_client_vm" { boot_disk { initialize_params { - image = "debian-cloud/debian-12" + image = "debian-cloud/debian-13" size = 80 } } @@ -369,194 +369,7 @@ resource "google_compute_instance" "orion_client_vm" { access_config {} } - metadata_startup_script = <<-EOT - #!/usr/bin/env bash - set -euo pipefail - - exec > >(tee -a /var/log/orion-client-startup.log) 2>&1 - - export DEBIAN_FRONTEND=noninteractive - - apt-get update - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - git \ - git-lfs \ - gettext-base \ - netcat-openbsd \ - procps \ - zstd \ - fuse3 \ - libfuse3-3 \ - libssl3 \ - build-essential \ - pkg-config \ - cmake \ - clang \ - llvm-dev \ - libclang-dev \ - libssl-dev \ - libfuse3-dev \ - protobuf-compiler - - echo "user_allow_other" >> /etc/fuse.conf || true - - mkdir -p \ - /opt/orion-client \ - /opt/orion-client/src \ - /opt/orion-client/bin \ - /opt/orion-client/log \ - /data/scorpio/store \ - /data/scorpio/antares/upper \ - /data/scorpio/antares/cl \ - /data/scorpio/antares/mnt \ - /workspace/mount - - # Install Rust toolchain if missing - if ! command -v rustc >/dev/null 2>&1; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - fi - export PATH="/root/.cargo/bin:$${PATH}" - - # Install Buck2 - BUCK2_VERSION="2025-06-01" - ARCH="x86_64-unknown-linux-musl" - curl -fsSL -o /usr/local/bin/buck2.zst "https://github.com/facebook/buck2/releases/download/$${BUCK2_VERSION}/buck2-$${ARCH}.zst" - zstd -d /usr/local/bin/buck2.zst -o /usr/local/bin/buck2 - chmod +x /usr/local/bin/buck2 - - # Create wrapper that behaves like the container entrypoint (start embedded scorpio then orion) - cat >/usr/local/bin/orion-worker-wrapper <<'WRAPPER' - #!/usr/bin/env bash - set -euo pipefail - - log() { echo "[orion-worker-wrapper] $*"; } - - : "${SERVER_WS:?SERVER_WS is required}" - - # Defaults aligned with mega/orion/entrypoint.sh - : "${SCORPIO_API_BASE_URL:=http://127.0.0.1:2725}" - : "${SCORPIO_HTTP_ADDR:=0.0.0.0:2725}" - : "${ORION_WORKER_START_SCORPIO:=true}" - - : "${SCORPIO_STORE_PATH:=/data/scorpio/store}" - : "${SCORPIO_WORKSPACE:=/workspace/mount}" - - mkdir -p "$SCORPIO_STORE_PATH" "$SCORPIO_WORKSPACE" \ - /data/scorpio/antares/upper /data/scorpio/antares/cl /data/scorpio/antares/mnt - - if [ "$ORION_WORKER_START_SCORPIO" != "false" ] && [ "$ORION_WORKER_START_SCORPIO" != "0" ]; then - if [ ! -e /dev/fuse ]; then - log "ERROR: /dev/fuse not found; Scorpio requires FUSE" - exit 1 - fi - - log "Starting embedded scorpio..." - # Use the config file prepared during installation - scorpio -c /etc/scorpio/scorpio.toml --http-addr "$SCORPIO_HTTP_ADDR" & - scorpio_pid=$! - - # Wait for scorpio to listen - port="${SCORPIO_HTTP_ADDR##*:}" - for i in $(seq 1 60); do - if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then - break - fi - sleep 1 - done - - if ! nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then - log "ERROR: scorpio did not become ready on 127.0.0.1:$port" - kill "$scorpio_pid" 2>/dev/null || true - exit 1 - fi - - log "Scorpio ready on 127.0.0.1:$port" - export SCORPIO_API_BASE_URL="http://127.0.0.1:$port" - fi - - log "Starting orion worker..." - exec orion -WRAPPER - chmod +x /usr/local/bin/orion-worker-wrapper - - # Install/build script: user uploads TGZ to /opt/orion-client/src.tgz, then runs this script. - cat >/opt/orion-client/install_from_tgz.sh <<'SCRIPT' - #!/usr/bin/env bash - set -euo pipefail - - TGZ_PATH="${1:-/opt/orion-client/src.tgz}" - SRC_DIR="/opt/orion-client/src" - - if [ ! -f "$TGZ_PATH" ]; then - echo "Error: Tarball not found at $TGZ_PATH" - exit 1 - fi - - rm -rf "$SRC_DIR"/* - - echo "Extracting source..." - # Note: no --strip-components because the tgz root is the workspace root - tar -xzf "$TGZ_PATH" -C "$SRC_DIR" - - echo "Preparing scorpio config..." - mkdir -p /etc/scorpio - if [ -f "$SRC_DIR/scorpio/scorpio.toml" ]; then - cp -f "$SRC_DIR/scorpio/scorpio.toml" /etc/scorpio/scorpio.toml - fi - - echo "Building orion and scorpio (release)..." - cd "$SRC_DIR" - export PATH="/root/.cargo/bin:$PATH" - - # Use a stable target dir to speed up rebuilds across reboots - export CARGO_TARGET_DIR=/opt/orion-client/target - - cargo build --release --package orion --package scorpio - - echo "Installing binaries..." - install -m 0755 "$CARGO_TARGET_DIR/release/orion" /usr/local/bin/orion - install -m 0755 "$CARGO_TARGET_DIR/release/scorpio" /usr/local/bin/scorpio - - echo "Writing systemd unit..." - cat >/etc/systemd/system/orion-worker.service < >(tee -a /var/log/orion-client-startup.log) 2>&1 + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + git-lfs \ + gettext-base \ + netcat-openbsd \ + procps \ + zstd \ + fuse3 \ + libssl3 \ + build-essential \ + pkg-config \ + cmake \ + clang \ + llvm-dev \ + libclang-dev \ + libssl-dev \ + libfuse3-dev \ + protobuf-compiler + +echo "user_allow_other" >> /etc/fuse.conf || true + +mkdir -p \ + /opt/orion-client \ + /opt/orion-client/src \ + /opt/orion-client/bin \ + /opt/orion-client/log \ + /data/scorpio/store \ + /data/scorpio/antares/upper \ + /data/scorpio/antares/cl \ + /data/scorpio/antares/mnt \ + /workspace/mount + +# Install Rust toolchain if missing +if ! command -v rustc >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +fi +export PATH="/root/.cargo/bin:${PATH}" + +# Install Buck2 +BUCK2_VERSION="2025-06-01" +ARCH="x86_64-unknown-linux-musl" +curl -fsSL -o /usr/local/bin/buck2.zst "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/buck2-${ARCH}.zst" +zstd -d /usr/local/bin/buck2.zst -o /usr/local/bin/buck2 +chmod +x /usr/local/bin/buck2 + +# Create wrapper that behaves like the container entrypoint (start embedded scorpio then orion) +cat >/usr/local/bin/orion-worker-wrapper <<'WRAPPER' +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[orion-worker-wrapper] $*"; } + +: "${SERVER_WS:?SERVER_WS is required}" + +# Defaults aligned with mega/orion/entrypoint.sh +: "${SCORPIO_API_BASE_URL:=http://127.0.0.1:2725}" +: "${SCORPIO_HTTP_ADDR:=0.0.0.0:2725}" +: "${ORION_WORKER_START_SCORPIO:=true}" + +: "${SCORPIO_STORE_PATH:=/data/scorpio/store}" +: "${SCORPIO_WORKSPACE:=/workspace/mount}" + +mkdir -p "$SCORPIO_STORE_PATH" "$SCORPIO_WORKSPACE" \ + /data/scorpio/antares/upper /data/scorpio/antares/cl /data/scorpio/antares/mnt + +if [ "$ORION_WORKER_START_SCORPIO" != "false" ] && [ "$ORION_WORKER_START_SCORPIO" != "0" ]; then + if [ ! -e /dev/fuse ]; then + log "ERROR: /dev/fuse not found; Scorpio requires FUSE" + exit 1 + fi + + log "Starting embedded scorpio..." + scorpio -c /etc/scorpio/scorpio.toml --http-addr "$SCORPIO_HTTP_ADDR" & + scorpio_pid=$! + + # Wait for scorpio to listen + port="${SCORPIO_HTTP_ADDR##*:}" + for i in $(seq 1 60); do + if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + if ! nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then + log "ERROR: scorpio did not become ready on 127.0.0.1:$port" + kill "$scorpio_pid" 2>/dev/null || true + exit 1 + fi + + log "Scorpio ready on 127.0.0.1:$port" + export SCORPIO_API_BASE_URL="http://127.0.0.1:$port" +fi + +log "Starting orion worker..." +exec orion +WRAPPER + +chmod +x /usr/local/bin/orion-worker-wrapper + +# Write and enable systemd service for orion worker +cat >/etc/systemd/system/orion-worker.service <<'EOF' +[Unit] +Description=Orion Worker Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=root +Environment="SERVER_WS=wss://buck2hub-orion-504513835593.asia-east1.run.app/ws" +Environment="SCORPIO_BASE_URL=https://buck2hub-mono-504513835593.asia-east1.run.app" +Environment="SCORPIO_LFS_URL=https://buck2hub-mono-504513835593.asia-east1.run.app" +Environment="SCORPIO_STORE_PATH=/data/scorpio/store" +Environment="SCORPIO_WORKSPACE=/workspace/mount" +Environment="ORION_WORKER_START_SCORPIO=true" +Environment="SCORPIO_HTTP_ADDR=0.0.0.0:2725" +ExecStart=/usr/local/bin/orion-worker-wrapper +Restart=always +RestartSec=5 +LimitNOFILE=1048576 + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable orion-worker.service +