diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f20b1a074..66dab17c7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + CONTAINER_TOOL: docker + jobs: changes: if: github.repository_owner == 'jumpstarter-dev' @@ -33,14 +36,147 @@ jobs: - '.github/workflows/e2e.yaml' - 'Makefile' - e2e-tests: + # =========================================================================== + # Build jobs: container images and Python wheels built in parallel + # =========================================================================== + + build-controller-image: + needs: changes + if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + strategy: + matrix: + include: + - os: ubuntu-24.04 + arch: amd64 + - os: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + + - name: Cache controller image + id: cache + uses: actions/cache@v4 + with: + path: /tmp/controller-image.tar + key: controller-image-${{ matrix.arch }}-${{ hashFiles('controller/Makefile', 'controller/Dockerfile', 'controller/go.mod', 'controller/go.sum', 'controller/cmd/**', 'controller/api/**', 'controller/internal/**') }} + + - name: Build controller image + if: steps.cache.outputs.cache-hit != 'true' + run: | + make -C controller docker-build + docker save quay.io/jumpstarter-dev/jumpstarter-controller:latest -o /tmp/controller-image.tar + + - name: Upload controller image + uses: actions/upload-artifact@v4 + with: + name: controller-image-${{ matrix.arch }} + path: /tmp/controller-image.tar + retention-days: 1 + + build-operator-image: + needs: changes + if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + strategy: + matrix: + include: + - os: ubuntu-24.04 + arch: amd64 + - os: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + + - name: Cache operator artifacts + id: cache + uses: actions/cache@v4 + with: + path: | + /tmp/operator-image.tar + /tmp/operator-install.yaml + key: operator-image-${{ matrix.arch }}-${{ hashFiles('controller/Makefile', 'controller/Dockerfile.operator', 'controller/go.mod', 'controller/go.sum', 'controller/deploy/operator/**', 'controller/api/**', 'controller/internal/**') }} + + - name: Build operator image and installer manifest + if: steps.cache.outputs.cache-hit != 'true' + run: | + make -C controller/deploy/operator docker-build build-installer VERSION=latest + docker save quay.io/jumpstarter-dev/jumpstarter-operator:latest -o /tmp/operator-image.tar + cp controller/deploy/operator/dist/install.yaml /tmp/operator-install.yaml + + - name: Upload operator artifacts + uses: actions/upload-artifact@v4 + with: + name: operator-image-${{ matrix.arch }} + path: | + /tmp/operator-image.tar + /tmp/operator-install.yaml + retention-days: 1 + + build-python-wheels: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Cache python wheels + id: cache + uses: actions/cache@v4 + with: + path: python/dist + key: python-wheels-${{ hashFiles('python/**/*.py', 'python/**/pyproject.toml', 'python/uv.lock') }} + + - name: Build python wheels + if: steps.cache.outputs.cache-hit != 'true' + working-directory: python + run: uv build --all --out-dir dist + + - name: Upload python wheels + uses: actions/upload-artifact@v4 + with: + name: python-wheels + path: python/dist/ + retention-days: 1 + + # =========================================================================== + # E2E test jobs: depend on pre-built artifacts + # =========================================================================== + + e2e-tests: + needs: [build-controller-image, build-operator-image, build-python-wheels] strategy: matrix: - os: - - ubuntu-24.04 - - ubuntu-24.04-arm + include: + - os: ubuntu-24.04 + arch: amd64 + - os: ubuntu-24.04-arm + arch: arm64 runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -55,11 +191,39 @@ jobs: with: go-version: "1.22" + - name: Download controller image + uses: actions/download-artifact@v4 + with: + name: controller-image-${{ matrix.arch }} + path: /tmp/artifacts + + - name: Download operator image + uses: actions/download-artifact@v4 + with: + name: operator-image-${{ matrix.arch }} + path: /tmp/artifacts + + - name: Download python wheels + uses: actions/download-artifact@v4 + with: + name: python-wheels + path: /tmp/python-wheels + + - name: Load container images and operator manifest + run: | + docker load < /tmp/artifacts/controller-image.tar + docker load < /tmp/artifacts/operator-image.tar + mkdir -p controller/deploy/operator/dist + cp /tmp/artifacts/operator-install.yaml controller/deploy/operator/dist/install.yaml + - name: Setup e2e test environment run: make e2e-setup env: CI: true METHOD: operator + SKIP_BUILD: "true" + PREBUILT_WHEELS_DIR: /tmp/python-wheels + OPERATOR_IMG: quay.io/jumpstarter-dev/jumpstarter-operator:latest - name: Run e2e tests run: make e2e-run @@ -73,7 +237,7 @@ jobs: # ============================================================================ e2e-compat-old-controller: - needs: changes + needs: [changes, build-python-wheels] if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -89,11 +253,18 @@ jobs: with: go-version: "1.22" + - name: Download python wheels + uses: actions/download-artifact@v4 + with: + name: python-wheels + path: /tmp/python-wheels + - name: Setup compat environment (old controller v0.7.0) run: make e2e-compat-setup COMPAT_SCENARIO=old-controller env: CI: true COMPAT_CONTROLLER_TAG: v0.7.0 + PREBUILT_WHEELS_DIR: /tmp/python-wheels - name: Run compat tests (old controller + new client/exporter) run: make e2e-compat-run COMPAT_TEST=old-controller @@ -101,7 +272,7 @@ jobs: CI: true e2e-compat-old-client: - needs: changes + needs: [changes, build-controller-image, build-python-wheels] if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -117,11 +288,28 @@ jobs: with: go-version: "1.22" + - name: Download controller image + uses: actions/download-artifact@v4 + with: + name: controller-image-amd64 + path: /tmp/artifacts + + - name: Download python wheels + uses: actions/download-artifact@v4 + with: + name: python-wheels + path: /tmp/python-wheels + + - name: Load controller image + run: docker load < /tmp/artifacts/controller-image.tar + - name: Setup compat environment (old client v0.7.0) run: make e2e-compat-setup COMPAT_SCENARIO=old-client env: CI: true COMPAT_CLIENT_VERSION: "0.7.1" + SKIP_BUILD: "true" + PREBUILT_WHEELS_DIR: /tmp/python-wheels - name: Run compat tests (new controller + old client/exporter) run: make e2e-compat-run COMPAT_TEST=old-client diff --git a/controller/Makefile b/controller/Makefile index c21b3d26a..08c9aa00e 100644 --- a/controller/Makefile +++ b/controller/Makefile @@ -188,9 +188,14 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy -deploy: docker-build cluster grpcurl ## Deploy controller using METHOD (operator or helm). Set CLUSTER_TYPE=k3s for k3s. +deploy: cluster grpcurl ## Deploy controller using METHOD (operator or helm). Set SKIP_BUILD=1 to skip image builds. +ifeq ($(SKIP_BUILD),) + $(MAKE) docker-build +endif ifeq ($(METHOD),operator) +ifeq ($(SKIP_BUILD),) $(MAKE) build-operator +endif ./hack/deploy_with_operator.sh else ifeq ($(METHOD),helm) ./hack/deploy_with_helm.sh diff --git a/e2e/compat/setup.sh b/e2e/compat/setup.sh index 6907bca1e..43150f44d 100755 --- a/e2e/compat/setup.sh +++ b/e2e/compat/setup.sh @@ -55,83 +55,6 @@ is_ci() { [ -n "${CI:-}" ] || [ -n "${GITHUB_ACTIONS:-}" ] } -# Check if bats libraries are available -check_bats_libraries() { - if ! command -v bats &> /dev/null; then - return 1 - fi - - if ! bats --version &> /dev/null; then - return 1 - fi - - local test_file=$(mktemp) - cat > "$test_file" <<'EOF' -setup() { - bats_load_library bats-support - bats_load_library bats-assert -} - -@test "dummy" { - run echo "test" - assert_success -} -EOF - - if bats "$test_file" &> /dev/null; then - rm -f "$test_file" - return 0 - else - rm -f "$test_file" - return 1 - fi -} - -# Install bats libraries locally -install_bats_libraries_local() { - local LIB_DIR="$REPO_ROOT/.bats/lib" - local ORIGINAL_DIR="$PWD" - - log_info "Installing bats helper libraries to $LIB_DIR..." - - mkdir -p "$LIB_DIR" - cd "$LIB_DIR" - - if [ ! -d "bats-support" ]; then - log_info "Cloning bats-support..." - git clone --depth 1 https://github.com/bats-core/bats-support.git - else - log_info "bats-support already installed" - fi - - if [ ! -d "bats-assert" ]; then - log_info "Cloning bats-assert..." - git clone --depth 1 https://github.com/bats-core/bats-assert.git - else - log_info "bats-assert already installed" - fi - - if [ ! -d "bats-file" ]; then - log_info "Cloning bats-file..." - git clone --depth 1 https://github.com/bats-core/bats-file.git - else - log_info "bats-file already installed" - fi - - cd "$ORIGINAL_DIR" - - export BATS_LIB_PATH="$LIB_DIR:${BATS_LIB_PATH:-}" - - log_info "Bats libraries installed successfully" - - if check_bats_libraries; then - log_info "Libraries verified and working" - else - log_error "Libraries installed but verification failed" - exit 1 - fi -} - # Install dependencies install_dependencies() { log_info "Installing dependencies..." @@ -145,28 +68,6 @@ install_dependencies() { log_info "Installing Python 3.12..." uv python install 3.12 - if ! command -v bats &> /dev/null; then - log_info "Installing bats..." - if is_ci; then - sudo apt-get update - sudo apt-get install -y bats - elif [[ "$OSTYPE" == "darwin"* ]]; then - log_info "Installing bats-core via Homebrew..." - brew install bats-core - else - log_error "bats not found. Please install it manually." - exit 1 - fi - fi - - if ! check_bats_libraries; then - log_info "Installing bats libraries locally..." - install_bats_libraries_local - else - log_info "Bats libraries are already available" - export BATS_LIB_PATH="$REPO_ROOT/.bats/lib:${BATS_LIB_PATH:-}" - fi - log_info "Dependencies installed" } @@ -249,8 +150,11 @@ deploy_new_controller() { cd "$REPO_ROOT" - # Build image from HEAD and deploy with helm (no OIDC, no extra values) - make -C controller docker-build + if [ -z "${SKIP_BUILD:-}" ]; then + make -C controller docker-build + else + log_info "Skipping controller image build (SKIP_BUILD is set)" + fi cd controller ./hack/deploy_with_helm.sh cd "$REPO_ROOT" @@ -258,17 +162,9 @@ deploy_new_controller() { log_info "New controller deployed" } -# Install jumpstarter Python packages from HEAD -install_jumpstarter() { - log_info "Installing jumpstarter from HEAD..." - - cd "$REPO_ROOT" - cd python - make sync - cd .. - - log_info "Jumpstarter python installed" -} +# Install jumpstarter Python packages from HEAD (shared helper) +# shellcheck source=../lib/install.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/lib/install.sh" # Install old client from PyPI into separate venv install_old_client() { diff --git a/e2e/compat/tests-old-client.bats b/e2e/compat/tests-old-client.bats deleted file mode 100644 index 0468279ee..000000000 --- a/e2e/compat/tests-old-client.bats +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env bats -# Compatibility E2E tests: New Controller + Old Client/Exporter (v0.7.x) -# -# Tests that old client and exporter code (installed from PyPI) works -# correctly against the new controller. Verifies that old exporters -# are not incorrectly marked as offline by the new controller. -# -# Uses NEW jmp admin for setup (Kubernetes API) and OLD jmp ($OLD_JMP) -# for client/exporter operations (controller gRPC). - -JS_NAMESPACE="${JS_NAMESPACE:-jumpstarter-lab}" -OLD_JMP="${OLD_JMP:-}" - -# File to track bash wrapper process PIDs across tests -COMPAT_PIDS_FILE="${BATS_RUN_TMPDIR:-/tmp}/compat_old_client_pids.txt" - -setup_file() { - # Initialize the PIDs file at the start of all tests - echo "" > "$COMPAT_PIDS_FILE" - - # Verify OLD_JMP is set and exists - if [ -z "$OLD_JMP" ] || [ ! -x "$OLD_JMP" ]; then - echo "ERROR: OLD_JMP not set or not executable: '$OLD_JMP'" >&2 - echo "Please run setup.sh with COMPAT_SCENARIO=old-client first." >&2 - exit 1 - fi - - echo "Using OLD_JMP: $OLD_JMP" >&2 - echo "OLD_JMP version:" >&2 - $OLD_JMP version >&2 2>/dev/null || echo "(version command not supported)" >&2 -} - -setup() { - bats_load_library bats-support - bats_load_library bats-assert - - bats_require_minimum_version 1.5.0 -} - -teardown_file() { - echo "" >&2 - echo "========================================" >&2 - echo "COMPAT OLD-CLIENT TEARDOWN RUNNING" >&2 - echo "========================================" >&2 - - # Kill tracked PIDs - if [ -f "$COMPAT_PIDS_FILE" ]; then - while IFS= read -r pid; do - if [ -n "$pid" ]; then - if ps -p "$pid" > /dev/null 2>&1; then - echo " Killing PID $pid" >&2 - kill -9 "$pid" 2>/dev/null || true - fi - fi - done < "$COMPAT_PIDS_FILE" - fi - - # Kill any orphaned old jmp processes - pkill -9 -f "jmp run --exporter compat-old-" 2>/dev/null || true - - # Clean up the PIDs file - rm -f "$COMPAT_PIDS_FILE" - - # Clean up CRDs (use new admin CLI) - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-old-client --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-old-exporter --delete 2>/dev/null || true - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-old-client-wait --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-old-exporter-wait --delete 2>/dev/null || true - - echo "=== Compat old-client cleanup complete ===" >&2 -} - -wait_for_compat_exporter() { - # Brief delay to avoid catching pre-disconnect state - sleep 2 - kubectl -n "${JS_NAMESPACE}" wait --timeout 5m \ - --for=condition=Online --for=condition=Registered \ - exporters.jumpstarter.dev/compat-old-exporter -} - -stop_compat_exporter() { - # Kill the bash wrapper loop and any child jmp processes - if [ -f "$COMPAT_PIDS_FILE" ]; then - while IFS= read -r pid; do - if [ -n "$pid" ]; then - kill -9 "$pid" 2>/dev/null || true - fi - done < "$COMPAT_PIDS_FILE" - fi - pkill -9 -f "jmp run --exporter compat-old-exporter$" 2>/dev/null || true - # Clear tracked PIDs since they're now dead - echo "" > "$COMPAT_PIDS_FILE" -} - -# ============================================================================ -# Setup: Use NEW admin CLI to create resources -# ============================================================================ - -@test "compat-old-client: create resources" { - run jmp admin create client -n "${JS_NAMESPACE}" compat-old-client --unsafe --save - assert_success - - run jmp admin create exporter -n "${JS_NAMESPACE}" compat-old-exporter --save \ - --label example.com/board=compat-old - assert_success - - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/compat-old-exporter.yaml -} - -# ============================================================================ -# Old exporter registration and Online status -# ============================================================================ - -@test "compat-old-client: old exporter registers with new controller" { - cat <&- & -while true; do - $OLD_JMP run --exporter compat-old-exporter - sleep 2 -done -EOF - echo "$!" >> "$COMPAT_PIDS_FILE" - - wait_for_compat_exporter -} - -@test "compat-old-client: old exporter shows as Online (not incorrectly offline)" { - wait_for_compat_exporter - - # Key regression: the new controller must NOT mark old exporters offline - # due to missing hooks protocol fields in ReportStatus. - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-old-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Online")].status}' - assert_success - assert_output "True" - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-old-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Registered")].status}' - assert_success - assert_output "True" -} - -# ============================================================================ -# Lease cycles: old client, new client, and Online status after leases -# ============================================================================ - -@test "compat-old-client: old client can connect through new controller" { - wait_for_compat_exporter - - run $OLD_JMP shell --client compat-old-client \ - --selector example.com/board=compat-old j power on - assert_success -} - -@test "compat-old-client: old exporter stays Online after lease completes" { - # After a lease cycle, the old exporter won't send the new ReportStatus - # fields (status_version, previous_status, release_lease). The new controller - # must not mark the exporter offline because of this. - wait_for_compat_exporter - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-old-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Online")].status}' - assert_success - assert_output "True" -} - -@test "compat-old-client: new client can connect to old exporter" { - wait_for_compat_exporter - - run jmp shell --client compat-old-client \ - --selector example.com/board=compat-old j power on - assert_success -} - -@test "compat-old-client: old exporter still Online after multiple lease cycles" { - # Catches accumulated state drift across lease transitions - # when the controller doesn't receive new status fields. - wait_for_compat_exporter - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-old-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Online")].status}' - assert_success - assert_output "True" - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-old-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Registered")].status}' - assert_success - assert_output "True" -} - -# ============================================================================ -# Reconnect after offline: exporter must recover from stale Offline status -# ============================================================================ - -@test "compat-old-client: stop exporter and wait for offline" { - stop_compat_exporter - - # The reconciler marks Online=False after LastSeen is >1 minute stale - kubectl -n "${JS_NAMESPACE}" wait --timeout 5m \ - --for=condition=Online=False \ - exporters.jumpstarter.dev/compat-old-exporter -} - -@test "compat-old-client: old exporter recovers Online after reconnect" { - # Restart the exporter - cat <&- & -while true; do - $OLD_JMP run --exporter compat-old-exporter - sleep 2 -done -EOF - echo "$!" >> "$COMPAT_PIDS_FILE" - - # This is the key assertion: the exporter must become Online again. - # Without the fix, it stays stuck as Offline forever. - wait_for_compat_exporter -} - -@test "compat-old-client: lease works after reconnect" { - wait_for_compat_exporter - - run $OLD_JMP shell --client compat-old-client \ - --selector example.com/board=compat-old j power on - assert_success -} - -# ============================================================================ -# Client started before exporter -# ============================================================================ - -@test "compat-old-client: client started before exporter connects" { - # Stop any running wait exporters for a clean test - pkill -9 -f "jmp run --exporter compat-old-exporter-wait" 2>/dev/null || true - sleep 3 - - # Create fresh resources for this test - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-old-exporter-wait --delete 2>/dev/null || true - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-old-client-wait --delete 2>/dev/null || true - - run jmp admin create client -n "${JS_NAMESPACE}" compat-old-client-wait --unsafe --save - assert_success - - run jmp admin create exporter -n "${JS_NAMESPACE}" compat-old-exporter-wait --save \ - --label example.com/board=compat-old-wait - assert_success - - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/compat-old-exporter-wait.yaml - - # Start new client shell BEFORE old exporter is running (in background) - # The client should wait for a matching exporter to become available - jmp shell --client compat-old-client-wait \ - --selector example.com/board=compat-old-wait j power on & - local CLIENT_PID=$! - - # Wait a few seconds to ensure the client is actively waiting for lease fulfillment - sleep 5 - - # Verify the client is still waiting (hasn't exited yet) - if ! kill -0 $CLIENT_PID 2>/dev/null; then - wait $CLIENT_PID || true - fail "Client exited before exporter was started" - fi - - # Now start the old exporter - cat <&- & -while true; do - $OLD_JMP run --exporter compat-old-exporter-wait - sleep 2 -done -EOF - echo "$!" >> "$COMPAT_PIDS_FILE" - - # Wait for the client command to complete (with timeout) - local timeout=120 - local count=0 - while kill -0 $CLIENT_PID 2>/dev/null && [ $count -lt $timeout ]; do - sleep 1 - count=$((count + 1)) - done - - # Check if client process completed - if kill -0 $CLIENT_PID 2>/dev/null; then - kill -9 $CLIENT_PID 2>/dev/null || true - fail "Client shell timed out waiting for exporter (${timeout}s)" - fi - - # Verify client exited successfully - wait $CLIENT_PID - local exit_code=$? - [ $exit_code -eq 0 ] -} - -# ============================================================================ -# Cleanup -# ============================================================================ - -@test "compat-old-client: cleanup resources" { - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-old-client --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-old-exporter --delete 2>/dev/null || true - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-old-client-wait --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-old-exporter-wait --delete 2>/dev/null || true -} diff --git a/e2e/compat/tests-old-controller.bats b/e2e/compat/tests-old-controller.bats deleted file mode 100644 index 0fc46197e..000000000 --- a/e2e/compat/tests-old-controller.bats +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env bats -# Compatibility E2E tests: Old Controller (v0.7.x) + New Client/Exporter -# -# Tests that the new client and exporter code works correctly against -# an older controller version that doesn't support hooks protocol changes. -# Also includes the client-started-before-exporter scenario. - -JS_NAMESPACE="${JS_NAMESPACE:-jumpstarter-lab}" - -# File to track bash wrapper process PIDs across tests -COMPAT_PIDS_FILE="${BATS_RUN_TMPDIR:-/tmp}/compat_old_ctrl_pids.txt" - -setup_file() { - # Initialize the PIDs file at the start of all tests - echo "" > "$COMPAT_PIDS_FILE" -} - -setup() { - bats_load_library bats-support - bats_load_library bats-assert - - bats_require_minimum_version 1.5.0 -} - -teardown_file() { - echo "" >&2 - echo "========================================" >&2 - echo "COMPAT OLD-CONTROLLER TEARDOWN RUNNING" >&2 - echo "========================================" >&2 - - # Kill tracked PIDs - if [ -f "$COMPAT_PIDS_FILE" ]; then - while IFS= read -r pid; do - if [ -n "$pid" ]; then - if ps -p "$pid" > /dev/null 2>&1; then - echo " Killing PID $pid" >&2 - kill -9 "$pid" 2>/dev/null || true - fi - fi - done < "$COMPAT_PIDS_FILE" - fi - - # Kill any orphaned jmp processes for compat exporters - pkill -9 -f "jmp run --exporter compat-" 2>/dev/null || true - - # Clean up the PIDs file - rm -f "$COMPAT_PIDS_FILE" - - # Clean up CRDs (best effort) - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-client --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-exporter --delete 2>/dev/null || true - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-client-wait --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-exporter-wait --delete 2>/dev/null || true - - echo "=== Compat old-controller cleanup complete ===" >&2 -} - -wait_for_compat_exporter() { - # Brief delay to avoid catching pre-disconnect state - sleep 2 - kubectl -n "${JS_NAMESPACE}" wait --timeout 5m \ - --for=condition=Online --for=condition=Registered \ - exporters.jumpstarter.dev/compat-exporter -} - -# ============================================================================ -# Core compatibility tests: Old Controller + New Client/Exporter -# ============================================================================ - -@test "compat-old-ctrl: can create client with admin cli" { - run jmp admin create client -n "${JS_NAMESPACE}" compat-client --unsafe --save - assert_success -} - -@test "compat-old-ctrl: can create exporter with admin cli" { - run jmp admin create exporter -n "${JS_NAMESPACE}" compat-exporter --save \ - --label example.com/board=compat - assert_success - - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/compat-exporter.yaml -} - -@test "compat-old-ctrl: new exporter registers with old controller" { - cat <&- & -while true; do - jmp run --exporter compat-exporter - sleep 2 -done -EOF - echo "$!" >> "$COMPAT_PIDS_FILE" - - wait_for_compat_exporter -} - -@test "compat-old-ctrl: exporter shows as Online and Registered" { - wait_for_compat_exporter - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Online")].status}' - assert_success - assert_output "True" - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Registered")].status}' - assert_success - assert_output "True" -} - -@test "compat-old-ctrl: new client can lease and connect through old controller" { - wait_for_compat_exporter - - run jmp shell --client compat-client \ - --selector example.com/board=compat j power on - assert_success -} - -@test "compat-old-ctrl: can operate on leases through old controller" { - wait_for_compat_exporter - - jmp config client use compat-client - - jmp create lease --selector example.com/board=compat --duration 1d - jmp get leases - jmp get exporters - - # Verify label selector filtering works - run jmp get leases --selector example.com/board=compat -o yaml - assert_success - assert_output --partial "example.com/board=compat" - - run jmp get leases --selector example.com/board=doesnotexist - assert_success - assert_output "No resources found." - - jmp delete leases --all -} - -@test "compat-old-ctrl: exporter stays Online after lease cycle" { - # After lease operations, the new exporter sends ReportStatus with - # new fields (status_version, previous_status, release_lease) that - # the old controller doesn't understand. The exporter must remain Online. - wait_for_compat_exporter - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Online")].status}' - assert_success - assert_output "True" - - run kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/compat-exporter \ - -o jsonpath='{.status.conditions[?(@.type=="Registered")].status}' - assert_success - assert_output "True" -} - -# ============================================================================ -# Client started before exporter -# ============================================================================ - -@test "compat-old-ctrl: client started before exporter connects" { - # Stop any running compat exporters for a clean test - pkill -9 -f "jmp run --exporter compat-exporter-wait" 2>/dev/null || true - sleep 3 - - # Create fresh resources for this test - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-exporter-wait --delete 2>/dev/null || true - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-client-wait --delete 2>/dev/null || true - - run jmp admin create client -n "${JS_NAMESPACE}" compat-client-wait --unsafe --save - assert_success - - run jmp admin create exporter -n "${JS_NAMESPACE}" compat-exporter-wait --save \ - --label example.com/board=compat-wait - assert_success - - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/compat-exporter-wait.yaml - - # Start client shell BEFORE exporter is running (in background) - # The client should wait for a matching exporter to become available - jmp shell --client compat-client-wait \ - --selector example.com/board=compat-wait j power on & - local CLIENT_PID=$! - - # Wait a few seconds to ensure the client is actively waiting for lease fulfillment - sleep 5 - - # Verify the client is still waiting (hasn't exited yet) - if ! kill -0 $CLIENT_PID 2>/dev/null; then - wait $CLIENT_PID || true - fail "Client exited before exporter was started" - fi - - # Now start the exporter - cat <&- & -while true; do - jmp run --exporter compat-exporter-wait - sleep 2 -done -EOF - echo "$!" >> "$COMPAT_PIDS_FILE" - - # Wait for the client command to complete (with timeout) - local timeout=120 - local count=0 - while kill -0 $CLIENT_PID 2>/dev/null && [ $count -lt $timeout ]; do - sleep 1 - count=$((count + 1)) - done - - # Check if client process completed - if kill -0 $CLIENT_PID 2>/dev/null; then - kill -9 $CLIENT_PID 2>/dev/null || true - fail "Client shell timed out waiting for exporter (${timeout}s)" - fi - - # Verify client exited successfully - wait $CLIENT_PID - local exit_code=$? - [ $exit_code -eq 0 ] -} - -# ============================================================================ -# Cleanup -# ============================================================================ - -@test "compat-old-ctrl: cleanup resources" { - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-client --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-exporter --delete 2>/dev/null || true - jmp admin delete client --namespace "${JS_NAMESPACE}" compat-client-wait --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" compat-exporter-wait --delete 2>/dev/null || true -} diff --git a/e2e/lib/install.sh b/e2e/lib/install.sh new file mode 100644 index 000000000..409f1e1b2 --- /dev/null +++ b/e2e/lib/install.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Shared installation helpers for e2e setup scripts. +# Requires: log_info, log_error functions and REPO_ROOT variable. + +# install_jumpstarter installs Python packages either from pre-built wheels +# (when PREBUILT_WHEELS_DIR is set) or via make sync. +install_jumpstarter() { + log_info "Installing jumpstarter..." + + cd "$REPO_ROOT" + if [ -n "${PREBUILT_WHEELS_DIR:-}" ]; then + if [ ! -d "${PREBUILT_WHEELS_DIR}" ]; then + log_error "PREBUILT_WHEELS_DIR is set but directory does not exist: ${PREBUILT_WHEELS_DIR}" + exit 1 + fi + local whl_count + whl_count=$(find "${PREBUILT_WHEELS_DIR}" -maxdepth 1 -name '*.whl' | wc -l) + if [ "$whl_count" -eq 0 ]; then + log_error "PREBUILT_WHEELS_DIR contains no .whl files: ${PREBUILT_WHEELS_DIR}" + exit 1 + fi + log_info "Installing from pre-built wheels in ${PREBUILT_WHEELS_DIR} (${whl_count} wheels)..." + cd python + uv venv .venv --python 3.12 + uv pip install --python .venv/bin/python "${PREBUILT_WHEELS_DIR}"/*.whl + cd .. + else + cd python + make sync + cd .. + fi + log_info "✓ Jumpstarter python installed" +} diff --git a/e2e/setup-e2e.sh b/e2e/setup-e2e.sh index 14c6decae..fc8bce67f 100755 --- a/e2e/setup-e2e.sh +++ b/e2e/setup-e2e.sh @@ -39,95 +39,6 @@ is_ci() { [ -n "${CI:-}" ] || [ -n "${GITHUB_ACTIONS:-}" ] } -# Check if bats libraries are available -check_bats_libraries() { - if ! command -v bats &> /dev/null; then - return 1 - fi - - # Try to load the libraries - if ! bats --version &> /dev/null; then - return 1 - fi - - # Check if libraries can be loaded by testing with a simple script - local test_file=$(mktemp) - cat > "$test_file" <<'EOF' -setup() { - bats_load_library bats-support - bats_load_library bats-assert -} - -@test "dummy" { - run echo "test" - assert_success -} -EOF - - # Run test with current BATS_LIB_PATH - if bats "$test_file" &> /dev/null; then - rm -f "$test_file" - return 0 - else - rm -f "$test_file" - return 1 - fi -} - -# Install bats libraries locally (works on all systems) -install_bats_libraries_local() { - local LIB_DIR="$REPO_ROOT/.bats/lib" - local ORIGINAL_DIR="$PWD" - - log_info "Installing bats helper libraries to $LIB_DIR..." - - mkdir -p "$LIB_DIR" - cd "$LIB_DIR" - - # Install bats-support - if [ ! -d "bats-support" ]; then - log_info "Cloning bats-support..." - git clone --depth 1 https://github.com/bats-core/bats-support.git - else - log_info "bats-support already installed" - fi - - # Install bats-assert - if [ ! -d "bats-assert" ]; then - log_info "Cloning bats-assert..." - git clone --depth 1 https://github.com/bats-core/bats-assert.git - else - log_info "bats-assert already installed" - fi - - # Install bats-file - if [ ! -d "bats-file" ]; then - log_info "Cloning bats-file..." - git clone --depth 1 https://github.com/bats-core/bats-file.git - else - log_info "bats-file already installed" - fi - - cd "$ORIGINAL_DIR" - - # Set BATS_LIB_PATH - export BATS_LIB_PATH="$LIB_DIR:${BATS_LIB_PATH:-}" - - log_info "✓ Bats libraries installed successfully" - log_info "BATS_LIB_PATH set to: $BATS_LIB_PATH" - - # Verify installation worked - if check_bats_libraries; then - log_info "✓ Libraries verified and working" - else - log_error "Libraries installed but verification failed" - log_error "Please check that the following directories exist:" - log_error " $LIB_DIR/bats-support" - log_error " $LIB_DIR/bats-assert" - exit 1 - fi -} - # Step 1: Install dependencies install_dependencies() { log_info "Installing dependencies..." @@ -143,61 +54,147 @@ install_dependencies() { log_info "Installing Python 3.12..." uv python install 3.12 - # Install bats if not already installed - if ! command -v bats &> /dev/null; then - log_info "Installing bats..." - if is_ci; then - sudo apt-get update - sudo apt-get install -y bats - elif [[ "$OSTYPE" == "darwin"* ]]; then - log_info "Installing bats-core via Homebrew..." - brew install bats-core - else - log_error "bats not found. Please install it manually:" - log_error " Ubuntu/Debian: sudo apt-get install bats" - log_error " Fedora/RHEL: sudo dnf install bats" - log_error " macOS: brew install bats-core" - exit 1 - fi + log_info "✓ Dependencies installed" +} + +# Step 2: Install e2e tools (cfssl, cfssljson, yq) as prebuilt binaries +E2E_TOOLS_BIN="$REPO_ROOT/.e2e/bin" +CFSSL_VERSION="1.6.5" +YQ_VERSION="v4.52.5" + +# SHA256 checksums for prebuilt binaries (from upstream release assets) +get_expected_sha256() { + case "$1" in + cfssl_linux_amd64) echo "ff4d3a1387ea3e1ee74f4bb8e5ffe9cbab5bee43c710333c206d14199543ebdf" ;; + cfssl_linux_arm64) echo "bc1a0b3a33ab415f3532af1d52cad7c9feec0156df2069f1cbbb64255485f108" ;; + cfssl_darwin_amd64) echo "6625b252053d9499bf26102b8fa78d7f675de56703d0808f8ff6dcf43121fa0c" ;; + cfssl_darwin_arm64) echo "9a38b997ac23bc2eed89d6ad79ea5ae27c29710f66fdabdff2aa16eaaadc30d4" ;; + cfssljson_linux_amd64) echo "09fbcb7a3b3d6394936ea61eabff1e8a59a8ac3b528deeb14cf66cdbbe9a534f" ;; + cfssljson_linux_arm64) echo "a389793bc2376116fe2fff996b4a2f772a59a4f65048a5cfb4789b2c0ea4a7c9" ;; + cfssljson_darwin_amd64) echo "1529a7a163801be8cf7d7a347b0346cc56cc8f351dbc0131373b6fb76bb4ab64" ;; + cfssljson_darwin_arm64) echo "no-prebuilt-binary-available" ;; + yq_linux_amd64) echo "75d893a0d5940d1019cb7cdc60001d9e876623852c31cfc6267047bc31149fa9" ;; + yq_linux_arm64) echo "90fa510c50ee8ca75544dbfffed10c88ed59b36834df35916520cddc623d9aaa" ;; + yq_darwin_amd64) echo "6e399d1eb466860c3202d231727197fdce055888c5c7bec6964156983dd1559d" ;; + yq_darwin_arm64) echo "45a12e64d4bd8a31c72ee1b889e81f1b1110e801baad3d6f030c111db0068de0" ;; + *) echo "" ;; + esac +} + +verify_sha256() { + local file="$1" expected="$2" + local actual + actual=$(shasum -a 256 "$file" | awk '{print $1}') + if [ "$actual" != "$expected" ]; then + log_error "SHA256 mismatch for $file" + log_error " expected: $expected" + log_error " actual: $actual" + rm -f "$file" + return 1 fi - - # Always install bats libraries locally for consistency across all systems - # This ensures libraries work regardless of package manager or distribution - if ! check_bats_libraries; then - log_info "Installing bats libraries locally..." - install_bats_libraries_local - else - log_info "✓ Bats libraries are already available" - # Still set BATS_LIB_PATH to include local directory for consistency - export BATS_LIB_PATH="$REPO_ROOT/.bats/lib:${BATS_LIB_PATH:-}" +} + +try_download_verified() { + local dest="$1" url="$2" hash_key="$3" + local expected_hash + expected_hash=$(get_expected_sha256 "$hash_key") + if [ -z "$expected_hash" ]; then + return 1 fi - - log_info "✓ Dependencies installed" + if curl -fsSL -o "$dest" "$url" && verify_sha256 "$dest" "$expected_hash"; then + chmod +x "$dest" + return 0 + fi + rm -f "$dest" + return 1 +} + +download_or_go_install() { + local name="$1" url="$2" go_pkg="$3" hash_key="$4" + local dest="$E2E_TOOLS_BIN/$name" + if [ -x "$dest" ]; then + return 0 + fi + log_info "Downloading ${name}..." + if try_download_verified "$dest" "$url" "$hash_key"; then + return 0 + fi + # On darwin/arm64, try the amd64 binary via Rosetta before compiling + if [ "$(uname -s)" = "Darwin" ] && [ "$(uname -m)" = "arm64" ]; then + local amd64_url="${url/darwin_arm64/darwin_amd64}" + local amd64_key="${hash_key/darwin_arm64/darwin_amd64}" + if [ "$amd64_url" != "$url" ]; then + log_info "Trying amd64 binary via Rosetta for ${name}..." + if try_download_verified "$dest" "$amd64_url" "$amd64_key"; then + return 0 + fi + fi + fi + log_warn "No verified prebuilt binary available, falling back to go install for ${name}..." + rm -f "$dest" + GOBIN="$E2E_TOOLS_BIN" go install "$go_pkg" } -# Step 2: Deploy dex +install_e2e_tools() { + log_info "Installing e2e tools..." + mkdir -p "$E2E_TOOLS_BIN" + + local arch + case "$(uname -m)" in + x86_64) arch="amd64" ;; + aarch64|arm64) arch="arm64" ;; + *) log_error "Unsupported architecture: $(uname -m)"; exit 1 ;; + esac + + local os + case "$(uname -s)" in + Linux) os="linux" ;; + Darwin) os="darwin" ;; + *) log_error "Unsupported OS: $(uname -s)"; exit 1 ;; + esac + + download_or_go_install cfssl \ + "https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssl_${CFSSL_VERSION}_${os}_${arch}" \ + "github.com/cloudflare/cfssl/cmd/cfssl@v${CFSSL_VERSION}" \ + "cfssl_${os}_${arch}" + + download_or_go_install cfssljson \ + "https://github.com/cloudflare/cfssl/releases/download/v${CFSSL_VERSION}/cfssljson_${CFSSL_VERSION}_${os}_${arch}" \ + "github.com/cloudflare/cfssl/cmd/cfssljson@v${CFSSL_VERSION}" \ + "cfssljson_${os}_${arch}" + + download_or_go_install yq \ + "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_${os}_${arch}" \ + "github.com/mikefarah/yq/v4@${YQ_VERSION}" \ + "yq_${os}_${arch}" + + export PATH="$E2E_TOOLS_BIN:$PATH" + log_info "✓ e2e tools installed to $E2E_TOOLS_BIN" +} + +# Step 3: Deploy dex deploy_dex() { log_info "Deploying dex..." cd "$REPO_ROOT" - # Generate certificates + # Generate certificates using prebuilt cfssl binaries log_info "Generating certificates..." - go run github.com/cloudflare/cfssl/cmd/cfssl@latest gencert -initca "$SCRIPT_DIR"/ca-csr.json | \ - go run github.com/cloudflare/cfssl/cmd/cfssljson@latest -bare ca - - go run github.com/cloudflare/cfssl/cmd/cfssl@latest gencert -ca=ca.pem -ca-key=ca-key.pem \ + cfssl gencert -initca "$SCRIPT_DIR"/ca-csr.json | cfssljson -bare ca - + cfssl gencert -ca=ca.pem -ca-key=ca-key.pem \ -config="$SCRIPT_DIR"/ca-config.json -profile=www "$SCRIPT_DIR"/dex-csr.json | \ - go run github.com/cloudflare/cfssl/cmd/cfssljson@latest -bare server + cfssljson -bare server make -C controller cluster - # Create dex namespace and TLS secret + # Create dex namespace and TLS secret (idempotent) log_info "Creating dex namespace and secrets..." - kubectl create namespace dex + kubectl create namespace dex --dry-run=client -o yaml | kubectl apply -f - kubectl -n dex create secret tls dex-tls \ --cert=server.pem \ - --key=server-key.pem + --key=server-key.pem \ + --dry-run=client -o yaml | kubectl apply -f - # Create .e2e directory for configuration files log_info "Creating .e2e directory for local configuration..." @@ -208,22 +205,23 @@ deploy_dex() { cp "$SCRIPT_DIR"/values.kind.yaml "$REPO_ROOT/.e2e/values.kind.yaml" log_info "Injecting CA certificate into values..." - go run github.com/mikefarah/yq/v4@latest -i \ + yq -i \ '.jumpstarter-controller.config.authentication.jwt[0].issuer.certificateAuthority = load_str("ca.pem")' \ "$REPO_ROOT/.e2e/values.kind.yaml" log_info "✓ Values file with CA certificate created at .e2e/values.kind.yaml" - # Create OIDC reviewer binding (important!) + # Create OIDC reviewer binding (idempotent) log_info "Creating OIDC reviewer cluster role binding..." kubectl create clusterrolebinding oidc-reviewer \ --clusterrole=system:service-account-issuer-discovery \ - --group=system:unauthenticated + --group=system:unauthenticated \ + --dry-run=client -o yaml | kubectl apply -f - - # Install dex via helm + # Install dex via helm (upgrade --install is idempotent) log_info "Installing dex via helm..." helm repo add dex https://charts.dexidp.io - helm install --namespace dex --wait -f "$SCRIPT_DIR"/dex.values.yaml dex dex/dex + helm upgrade --install --namespace dex --wait -f "$SCRIPT_DIR"/dex.values.yaml dex dex/dex # Install CA certificate log_info "Installing CA certificate..." @@ -262,7 +260,7 @@ deploy_dex() { log_info "✓ Dex deployed" } -# Step 3: Deploy jumpstarter controller +# Step 4: Deploy jumpstarter controller deploy_controller() { log_info "Deploying jumpstarter controller (method: $METHOD)..." @@ -274,31 +272,29 @@ deploy_controller() { exit 1 fi - # Deploy with CA certificate + # Use parallel make on Linux (GNU Make 4+); macOS ships an old make without --output-sync + local make_parallel="" + if [ "$(uname -s)" = "Linux" ]; then + make_parallel="-j5 --output-sync=target" + fi + log_info "Deploying controller with CA certificate using $METHOD..." if [ "$METHOD" = "operator" ]; then - # For operator: use OPERATOR_USE_DEX to inject dex config directly - OPERATOR_USE_DEX=true DEX_CA_FILE="$REPO_ROOT/ca.pem" METHOD=$METHOD make -C controller deploy + OPERATOR_USE_DEX=true DEX_CA_FILE="$REPO_ROOT/ca.pem" METHOD=$METHOD \ + make -C controller deploy $make_parallel else - # For helm: use EXTRA_VALUES to pass the values file - EXTRA_VALUES="--values $REPO_ROOT/.e2e/values.kind.yaml" METHOD=$METHOD make -C controller deploy + EXTRA_VALUES="--values $REPO_ROOT/.e2e/values.kind.yaml" METHOD=$METHOD \ + make -C controller deploy $make_parallel fi log_info "✓ Controller deployed" } -# Step 4: Install jumpstarter -install_jumpstarter() { - log_info "Installing jumpstarter..." - - cd "$REPO_ROOT" - cd python - make sync - cd .. - log_info "✓ Jumpstarter python installed" -} +# Step 5: Install jumpstarter (shared helper) +# shellcheck source=lib/install.sh +source "$SCRIPT_DIR/lib/install.sh" -# Step 5: Setup test environment +# Step 6: Setup test environment setup_test_environment() { log_info "Setting up test environment..." @@ -333,10 +329,10 @@ setup_test_environment() { log_info "Exporters directory already exists and is writable" fi - # Create service accounts + # Create service accounts (idempotent) log_info "Creating service accounts..." - kubectl create -n "${JS_NAMESPACE}" sa test-client-sa - kubectl create -n "${JS_NAMESPACE}" sa test-exporter-sa + kubectl create -n "${JS_NAMESPACE}" sa test-client-sa --dry-run=client -o yaml | kubectl apply -f - + kubectl create -n "${JS_NAMESPACE}" sa test-exporter-sa --dry-run=client -o yaml | kubectl apply -f - # Create a marker file to indicate setup is complete echo "ENDPOINT=$ENDPOINT" > "$REPO_ROOT/.e2e-setup-complete" @@ -349,8 +345,8 @@ setup_test_environment() { echo "SSL_CERT_FILE=$REPO_ROOT/ca.pem" >> "$REPO_ROOT/.e2e-setup-complete" echo "REQUESTS_CA_BUNDLE=$REPO_ROOT/ca.pem" >> "$REPO_ROOT/.e2e-setup-complete" - # Save BATS_LIB_PATH for test runs - echo "BATS_LIB_PATH=$BATS_LIB_PATH" >> "$REPO_ROOT/.e2e-setup-complete" + # Export e2e tools bin so downstream scripts (tests) can find yq, cfssl, etc. + echo "export PATH=\"$E2E_TOOLS_BIN:\$PATH\"" >> "$REPO_ROOT/.e2e-setup-complete" log_info "✓ Test environment ready" } @@ -367,6 +363,9 @@ main() { install_dependencies echo "" + install_e2e_tools + echo "" + deploy_dex echo "" diff --git a/e2e/tests-direct-listener.bats b/e2e/tests-direct-listener.bats deleted file mode 100644 index 0fb62b6c2..000000000 --- a/e2e/tests-direct-listener.bats +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env bats -# E2E tests for direct TCP listener mode (no controller) - -SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" && pwd)" -EXPORTER_CONFIG="${SCRIPT_DIR}/exporters/exporter-direct-listener.yaml" -LISTENER_PORT=19090 -LISTENER_PID="" - -setup() { - bats_load_library bats-support - bats_load_library bats-assert - - bats_require_minimum_version 1.5.0 -} - -# Start the exporter in the background. -# $1 - config file (default: $EXPORTER_CONFIG) -# $2 - readiness: "grpc" waits via jmp shell (drains LogStream), -# "port" waits via nc -z (preserves LogStream queue) -# $3 - if set, redirect stderr to ${BATS_TEST_TMPDIR}/exporter.log -# $4 - passphrase (optional) -_start_exporter() { - local config="${1:-$EXPORTER_CONFIG}" - local readiness="${2:-grpc}" - local capture_logs="${3:-}" - local passphrase="${4:-}" - - local extra_args=() - if [ -n "$passphrase" ]; then - extra_args+=(--passphrase "$passphrase") - fi - - if [ -n "$capture_logs" ]; then - jmp run --exporter-config "$config" \ - --tls-grpc-listener "$LISTENER_PORT" \ - --tls-grpc-insecure "${extra_args[@]}" 2>"${BATS_TEST_TMPDIR}/exporter.log" & - else - jmp run --exporter-config "$config" \ - --tls-grpc-listener "$LISTENER_PORT" \ - --tls-grpc-insecure "${extra_args[@]}" & - fi - LISTENER_PID=$! - echo "$LISTENER_PID" > "${BATS_TEST_TMPDIR}/exporter.pid" - - local retries=30 - if [ "$readiness" = "port" ]; then - # TCP-only check: doesn't drain the LogStream queue, so hook output - # remains buffered for the test command to consume. - while ! nc -z 127.0.0.1 "$LISTENER_PORT" 2>/dev/null; do - retries=$((retries - 1)) - if [ "$retries" -le 0 ]; then - echo "Port $LISTENER_PORT did not become available" >&2 - return 1 - fi - sleep 0.5 - done - else - # Full gRPC check: ensures exporter is ready for commands. - # Drains LogStream queue (unsuitable for hook output tests). - local grpc_args=(--tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure) - if [ -n "$passphrase" ]; then - grpc_args+=(--passphrase "$passphrase") - fi - while ! jmp shell "${grpc_args[@]}" -- j --help >/dev/null 2>&1; do - retries=$((retries - 1)) - if [ "$retries" -le 0 ]; then - echo "Exporter did not become ready in time" >&2 - return 1 - fi - sleep 0.5 - done - fi -} - -start_exporter() { _start_exporter "$1" grpc; } -start_exporter_with_logs() { _start_exporter "$1" grpc logs; } -start_exporter_bg() { _start_exporter "$1" port; } -start_exporter_bg_with_logs() { _start_exporter "$1" port logs; } - -start_exporter_with_passphrase() { _start_exporter "${2:-$EXPORTER_CONFIG}" grpc "" "$1"; } - -stop_exporter() { - if [ -f "${BATS_TEST_TMPDIR}/exporter.pid" ]; then - local pid - pid=$(cat "${BATS_TEST_TMPDIR}/exporter.pid") - if [ -n "$pid" ] && ps -p "$pid" > /dev/null 2>&1; then - kill "$pid" 2>/dev/null || true - wait "$pid" 2>/dev/null || true - fi - rm -f "${BATS_TEST_TMPDIR}/exporter.pid" - fi -} - -teardown() { - stop_exporter -} - -@test "direct listener: exporter starts and client can connect" { - start_exporter - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure -- j power on - assert_success -} - -@test "direct listener: client can call multiple driver methods" { - start_exporter - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure -- j power on - assert_success - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure -- j power off - assert_success -} - -@test "direct listener: client without --tls-grpc-insecure fails against insecure server" { - start_exporter - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" -- j power on - assert_failure -} - -@test "direct listener hooks: beforeLease hook executes and j commands work" { - # Use start_exporter_bg (TCP-only readiness check) to avoid draining - # the LogStream queue before the test command connects. - start_exporter_bg "${SCRIPT_DIR}/exporters/exporter-direct-hooks-before.yaml" - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure \ - --exporter-logs -- j power off - assert_success - assert_output --partial "BEFORE_HOOK_DIRECT: executed" - assert_output --partial "BEFORE_HOOK_DIRECT: complete" -} - -@test "direct listener hooks: afterLease hook runs on exporter shutdown" { - start_exporter_bg_with_logs "${SCRIPT_DIR}/exporters/exporter-direct-hooks-both.yaml" - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure \ - --exporter-logs -- j power on - assert_success - assert_output --partial "BEFORE_HOOK_DIRECT: executed" - - # Stop the exporter (SIGTERM triggers _cleanup_after_lease). - # stop_exporter waits for the process to exit, so the log is complete. - stop_exporter - - # afterLease hook output should appear in the exporter's stderr log - run cat "${BATS_TEST_TMPDIR}/exporter.log" - assert_output --partial "AFTER_HOOK_DIRECT: executed" -} - -@test "direct listener passphrase: correct passphrase connects" { - start_exporter_with_passphrase "my-secret" - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure \ - --passphrase "my-secret" -- j power on - assert_success -} - -@test "direct listener passphrase: wrong passphrase is rejected" { - start_exporter_with_passphrase "my-secret" - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure \ - --passphrase "wrong" -- j power on - assert_failure -} - -@test "direct listener passphrase: missing passphrase is rejected" { - start_exporter_with_passphrase "my-secret" - - run jmp shell --tls-grpc "127.0.0.1:${LISTENER_PORT}" --tls-grpc-insecure \ - -- j power on - assert_failure -} diff --git a/e2e/tests-hooks.bats b/e2e/tests-hooks.bats deleted file mode 100755 index a4e00e197..000000000 --- a/e2e/tests-hooks.bats +++ /dev/null @@ -1,526 +0,0 @@ -#!/usr/bin/env bats -# E2E tests for hooks feature (beforeLease/afterLease) - -JS_NAMESPACE="${JS_NAMESPACE:-jumpstarter-lab}" - -# File to track bash wrapper process PIDs across tests -HOOKS_EXPORTER_PIDS_FILE="${BATS_RUN_TMPDIR:-/tmp}/hooks_exporter_pids.txt" - -# Track which config is currently active -CURRENT_HOOKS_CONFIG="" - -setup_file() { - # Initialize the PIDs file at the start of all tests - echo "" > "$HOOKS_EXPORTER_PIDS_FILE" - - # Create client and exporter for hooks tests - jmp admin create client -n "${JS_NAMESPACE}" test-client-hooks --unsafe --out "${BATS_RUN_TMPDIR}/test-client-hooks.yaml" \ - --oidc-username dex:test-client-hooks - - jmp admin create exporter -n "${JS_NAMESPACE}" test-exporter-hooks --out "${BATS_RUN_TMPDIR}/test-exporter-hooks.yaml" \ - --oidc-username dex:test-exporter-hooks \ - --label example.com/board=hooks - - jmp login --client test-client-hooks \ - --endpoint "$ENDPOINT" --namespace "${JS_NAMESPACE}" --name test-client-hooks \ - --issuer https://dex.dex.svc.cluster.local:5556 \ - --username test-client-hooks@example.com --password password --unsafe - - jmp login --exporter test-exporter-hooks \ - --endpoint "$ENDPOINT" --namespace "${JS_NAMESPACE}" --name test-exporter-hooks \ - --issuer https://dex.dex.svc.cluster.local:5556 \ - --username test-exporter-hooks@example.com --password password -} - -setup() { - bats_load_library bats-support - bats_load_library bats-assert - - bats_require_minimum_version 1.5.0 -} - -teardown() { - # Clean up temp files that may leak if assertions fail before in-test cleanup - rm -f /tmp/jumpstarter-e2e-hook-python.py - rm -f /tmp/jumpstarter-e2e-hook-script.sh -} - -teardown_file() { - echo "" >&2 - echo "========================================" >&2 - echo "HOOKS TESTS TEARDOWN_FILE RUNNING" >&2 - echo "========================================" >&2 - - stop_hooks_exporter - - # Clean up client and exporter CRDs - jmp admin delete client --namespace "${JS_NAMESPACE}" test-client-hooks --delete 2>/dev/null || true - jmp admin delete exporter --namespace "${JS_NAMESPACE}" test-exporter-hooks --delete 2>/dev/null || true - - # Clean up the PIDs file - rm -f "$HOOKS_EXPORTER_PIDS_FILE" - - echo "=== Hooks cleanup complete ===" >&2 -} - -# Helper: Stop hooks exporter processes -stop_hooks_exporter() { - echo "=== Stopping hooks exporter processes ===" >&2 - - # Read PIDs from file - if [ -f "$HOOKS_EXPORTER_PIDS_FILE" ]; then - while IFS= read -r pid; do - if [ -n "$pid" ]; then - echo "Checking PID $pid..." >&2 - if ps -p "$pid" > /dev/null 2>&1; then - echo " Killing PID $pid" >&2 - kill -9 "$pid" 2>/dev/null || true - fi - fi - done < "$HOOKS_EXPORTER_PIDS_FILE" - # Clear the file - echo "" > "$HOOKS_EXPORTER_PIDS_FILE" - fi - - # Kill any orphaned jmp processes for hooks exporter - pkill -9 -f "jmp run --exporter test-exporter-hooks" 2>/dev/null || true - - # Give time for cleanup - sleep 1 -} - -# Helper: Start hooks exporter with restart loop (normal mode) -start_hooks_exporter() { - local config_file="$1" - - stop_hooks_exporter - - # Clear any leftover hooks config from previous test, then merge new config - go run github.com/mikefarah/yq/v4@latest -i 'del(.hooks)' \ - /etc/jumpstarter/exporters/test-exporter-hooks.yaml - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/${config_file}\")" \ - /etc/jumpstarter/exporters/test-exporter-hooks.yaml - - cat <&- & -while true; do - jmp run --exporter test-exporter-hooks - sleep 2 -done -EOF - echo "$!" >> "$HOOKS_EXPORTER_PIDS_FILE" - - wait_for_hooks_exporter -} - -# Helper: Start hooks exporter without restart loop (for exit mode tests) -start_hooks_exporter_single() { - local config_file="$1" - - stop_hooks_exporter - - # Clear any leftover hooks config from previous test, then merge new config - go run github.com/mikefarah/yq/v4@latest -i 'del(.hooks)' \ - /etc/jumpstarter/exporters/test-exporter-hooks.yaml - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/${config_file}\")" \ - /etc/jumpstarter/exporters/test-exporter-hooks.yaml - - jmp run --exporter test-exporter-hooks & - echo "$!" >> "$HOOKS_EXPORTER_PIDS_FILE" - - wait_for_hooks_exporter -} - -# Helper: Wait for hooks exporter to be online and registered -wait_for_hooks_exporter() { - # Brief delay to avoid catching pre-connect state - sleep 2 - kubectl -n "${JS_NAMESPACE}" wait --timeout 5m --for=condition=Online --for=condition=Registered \ - exporters.jumpstarter.dev/test-exporter-hooks - - # Also wait for exporterStatus=Available to ensure the exporter has finished - # cleaning up any previous lease and is ready for new ones. (See #425) - local timeout=300 - local elapsed=0 - while [ $elapsed -lt $timeout ]; do - local status - status=$(kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-hooks \ - -o jsonpath='{.status.exporterStatus}' 2>/dev/null || echo "") - if [ "$status" = "Available" ]; then - return 0 - fi - sleep 0.5 - elapsed=$((elapsed + 1)) - done - - echo "Timed out waiting for test-exporter-hooks to reach Available status" >&2 - return 1 -} - -# Helper: Wait for hooks exporter to go offline -wait_for_hooks_exporter_offline() { - local max_wait=200 - local count=0 - - while [ $count -lt $max_wait ]; do - local status=$(kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-hooks \ - -o jsonpath='{.status.conditions[?(@.type=="Online")].status}' 2>/dev/null || echo "Unknown") - - if [ "$status" = "False" ] || [ "$status" = "Unknown" ]; then - echo "Exporter is offline" >&2 - return 0 - fi - - sleep 1 - count=$((count + 1)) - done - - echo "Timed out waiting for exporter to go offline" >&2 - return 1 -} - -# Helper: Check if exporter process is still running -exporter_process_running() { - if [ -f "$HOOKS_EXPORTER_PIDS_FILE" ]; then - while IFS= read -r pid; do - if [ -n "$pid" ] && ps -p "$pid" > /dev/null 2>&1; then - return 0 - fi - done < "$HOOKS_EXPORTER_PIDS_FILE" - fi - return 1 -} - -# ============================================================================ -# Group A: Basic Hook Execution -# ============================================================================ - -@test "hooks A1: beforeLease hook executes" { - start_hooks_exporter "exporter-hooks-before-only.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "BEFORE_HOOK_MARKER: executed" -} - -@test "hooks A2: afterLease hook executes" { - start_hooks_exporter "exporter-hooks-after-only.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "AFTER_HOOK_MARKER: executed" -} - -@test "hooks A3: both hooks execute in correct order" { - start_hooks_exporter "exporter-hooks-both.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "BEFORE_HOOK:" - assert_output --partial "AFTER_HOOK:" - - # Verify order: BEFORE should appear before AFTER in output - local before_pos=$(echo "$output" | grep -n "BEFORE_HOOK:" | head -1 | cut -d: -f1) - local after_pos=$(echo "$output" | grep -n "AFTER_HOOK:" | head -1 | cut -d: -f1) - - [ "$before_pos" -lt "$after_pos" ] -} - -# ============================================================================ -# Group B: beforeLease Failure Modes -# ============================================================================ - -@test "hooks B1: beforeLease onFailure=warn allows shell to proceed" { - start_hooks_exporter "exporter-hooks-before-fail-warn.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Shell should succeed despite hook failure - assert_success - # Use short substring to avoid Rich text wrapping breaking the match - assert_output --partial "HOOK_FAIL_WARN" - - # Exporter should still be available - wait_for_hooks_exporter -} - -@test "hooks B2: beforeLease onFailure=endLease fails shell" { - start_hooks_exporter "exporter-hooks-before-fail-endLease.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Shell should fail because hook failed - assert_failure - # endLease may drop connection before status propagates to client - assert_output --regexp "(beforeLease hook fail|Exporter shutting down|Connection to exporter lost)" - - # Exporter should still be available after failure - wait_for_hooks_exporter -} - -@test "hooks B3: beforeLease onFailure=exit shuts down exporter" { - start_hooks_exporter_single "exporter-hooks-before-fail-exit.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Shell should fail - error includes reason from exporter status - assert_failure - # Exporter exit may drop connection before status propagates to client - assert_output --regexp "(beforeLease hook fail|Exporter shutting down|Connection to exporter lost)" - - # Exporter process should have exited - sleep 2 - run exporter_process_running - assert_failure - - # Exporter should go offline - wait_for_hooks_exporter_offline -} - -# ============================================================================ -# Group C: afterLease Failure Modes -# ============================================================================ - -@test "hooks C1: afterLease onFailure=warn keeps exporter available" { - start_hooks_exporter "exporter-hooks-after-fail-warn.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Shell should succeed (afterLease runs after shell completes) - assert_success - # Use short substring to avoid Rich text wrapping breaking the match - assert_output --partial "HOOK_FAIL_WARN" - - # Exporter should still be available - wait_for_hooks_exporter -} - -@test "hooks C2: afterLease onFailure=exit shuts down exporter" { - start_hooks_exporter_single "exporter-hooks-after-fail-exit.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Shell may succeed (command completes before afterLease runs) or fail - # (exporter shuts down during lease teardown). Either outcome is valid. - # The key behavior is that the exporter process exits and goes offline. - - # Exporter process should have exited - sleep 5 - run exporter_process_running - assert_failure - - # Exporter should go offline - wait_for_hooks_exporter_offline -} - -# ============================================================================ -# Group D: Timeout Tests -# ============================================================================ - -@test "hooks D1: beforeLease timeout is treated as failure" { - start_hooks_exporter "exporter-hooks-timeout.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Hook should timeout but shell proceeds (onFailure=warn) - assert_success - assert_output --partial "HOOK_TIMEOUT: starting" - - # Exporter should still be available - wait_for_hooks_exporter -} - -# ============================================================================ -# Group E: j Commands in Hooks -# ============================================================================ - -@test "hooks E1: beforeLease can use j power on" { - start_hooks_exporter "exporter-hooks-both.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "BEFORE_HOOK: complete" - # The j power on command in beforeLease should have executed -} - -@test "hooks E2: afterLease can use j power off" { - start_hooks_exporter "exporter-hooks-both.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "AFTER_HOOK: complete" - # The j power off command in afterLease should have executed -} - -@test "hooks E3: environment variables are available in hooks" { - start_hooks_exporter "exporter-hooks-both.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - # LEASE_NAME and CLIENT_NAME should be set (not empty) - assert_output --partial "BEFORE_HOOK:" - # Verify env vars are expanded (check components separately since Rich may wrap the line) - [[ "$output" =~ lease=[0-9a-f-]+ ]] - [[ "$output" =~ client= ]] -} - -# ============================================================================ -# Group F: Custom Executors (exec field) -# ============================================================================ - -@test "hooks F1: exec /bin/bash runs bash-specific syntax" { - start_hooks_exporter "exporter-hooks-exec-bash.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "BASH_HOOK: complete" -} - -@test "hooks F2: .py file auto-detects Python and uses driver API" { - # Create a Python hook script that uses the Jumpstarter client API - cat > /tmp/jumpstarter-e2e-hook-python.py << 'PYEOF' -import os -from jumpstarter.utils.env import env - -lease = os.environ.get("LEASE_NAME", "unknown") -print(f"PYTHON_HOOK: lease={lease}") - -with env() as client: - client.power.on() - print("PYTHON_HOOK: driver API works") - -print("PYTHON_HOOK: complete") -PYEOF - - start_hooks_exporter "exporter-hooks-exec-python.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "PYTHON_HOOK: driver API works" - assert_output --partial "PYTHON_HOOK: complete" - - rm -f /tmp/jumpstarter-e2e-hook-python.py -} - -@test "hooks F3: script as file path executes the file" { - # Create a temporary script file on the exporter host - cat > /tmp/jumpstarter-e2e-hook-script.sh << 'SCRIPT' -#!/bin/sh -echo "SCRIPTFILE_HOOK: executed from file" -echo "SCRIPTFILE_HOOK: complete" -SCRIPT - chmod +x /tmp/jumpstarter-e2e-hook-script.sh - - start_hooks_exporter "exporter-hooks-exec-script-file.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - assert_output --partial "SCRIPTFILE_HOOK: complete" - - rm -f /tmp/jumpstarter-e2e-hook-script.sh -} - -# ============================================================================ -# Group G: Lease Timeout (no hooks) -# ============================================================================ - -@test "hooks G1: no hooks with lease timeout exits cleanly" { - start_hooks_exporter "exporter-hooks-none.yaml" - - run timeout 60 jmp shell --client test-client-hooks \ - --selector example.com/board=hooks --duration 10s -- sleep 30 - - # Should not produce an error (exit code may be non-zero from killed sleep) - refute_output --partial "Error:" -} - -@test "hooks G2: lease timeout during slow beforeLease hook exits cleanly" { - start_hooks_exporter "exporter-hooks-slow-before.yaml" - - # Lease (5s) expires before hook (8s sleep) completes - run timeout 60 jmp shell --client test-client-hooks \ - --selector example.com/board=hooks --duration 5s -- sleep 30 - - refute_output --partial "Error:" -} - -@test "hooks G3: lease timeout shortly after beforeLease hook exits cleanly" { - start_hooks_exporter "exporter-hooks-slow-before.yaml" - - # Hook takes ~8s, lease is 12s, shell runs briefly before lease expires - run timeout 60 jmp shell --client test-client-hooks \ - --selector example.com/board=hooks --duration 12s -- sleep 30 - - refute_output --partial "Error:" -} - -# ============================================================================ -# Group H: PR Regression Tests -# ============================================================================ - -@test "hooks H1: infrastructure messages not visible in client output" { - # Issue A1: Hook infra messages (Starting hook subprocess, Creating PTY, etc.) - # are logged at DEBUG level. At default INFO level they should never enter - # the LogStream deque, so clients should not see them. - start_hooks_exporter "exporter-hooks-before-only.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_success - # User output from hook should be visible - assert_output --partial "BEFORE_HOOK_MARKER: executed" - # Infrastructure messages should NOT be visible (they are at DEBUG level) - refute_output --partial "Starting hook subprocess" - refute_output --partial "Creating PTY" - refute_output --partial "Hook executed successfully" -} - -@test "hooks H2: beforeLease fail+exit does NOT run afterLease hook" { - # Issue E1: When beforeLease fails with on_failure=exit, skip_after_lease_hook - # is set to True, preventing afterLease hook from executing in handle_lease's - # finally block. - start_hooks_exporter_single "exporter-hooks-before-fail-exit-with-after.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - assert_failure - # afterLease hook should NOT have run - refute_output --partial "AFTER_SHOULD_NOT_RUN" - - # Exporter should exit - wait_for_hooks_exporter_offline -} - -@test "hooks H3: warning displayed when beforeLease hook fails with warn" { - # Issue E5: When beforeLease hook fails with on_failure=warn, shell.py detects - # HOOK_WARNING_PREFIX in the status message and prints a user-visible warning. - start_hooks_exporter "exporter-hooks-before-fail-warn.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Shell should succeed despite warning - assert_success - # Client should display warning from hook failure - assert_output --partial "Warning:" -} - -@test "hooks H4: warning displayed when afterLease hook fails with warn" { - # Issue E5: When afterLease hook fails with on_failure=warn, shell.py waits - # for AVAILABLE status, detects HOOK_WARNING_PREFIX, and prints the warning. - start_hooks_exporter "exporter-hooks-after-fail-warn.yaml" - - run jmp shell --client test-client-hooks --selector example.com/board=hooks j power on - - # Shell should complete normally - assert_success - # Client should display afterLease hook warning - assert_output --partial "Warning:" -} diff --git a/e2e/tests.bats b/e2e/tests.bats deleted file mode 100755 index 8ea2de2c8..000000000 --- a/e2e/tests.bats +++ /dev/null @@ -1,579 +0,0 @@ -JS_NAMESPACE="${JS_NAMESPACE:-jumpstarter-lab}" - -# File to track bash wrapper process PIDs across tests -EXPORTER_PIDS_FILE="${BATS_RUN_TMPDIR:-/tmp}/exporter_pids.txt" - -# Directory for exporter log files -EXPORTER_LOGS_DIR="${BATS_RUN_TMPDIR:-/tmp}/exporter_logs" - -setup_file() { - # Initialize the PIDs file at the start of all tests - echo "" > "$EXPORTER_PIDS_FILE" - # Create directory for exporter logs - mkdir -p "$EXPORTER_LOGS_DIR" -} - -setup() { - bats_load_library bats-support - bats_load_library bats-assert - - bats_require_minimum_version 1.5.0 - - # Write test markers to exporter log files for easier correlation - local marker="=== TEST START: ${BATS_TEST_NAME} @ $(date -Iseconds) ===" - for logfile in "$EXPORTER_LOGS_DIR"/test-exporter-*.log; do - if [ -f "$logfile" ]; then - echo "$marker" >> "$logfile" - fi - done -} - -# Dump debug logs when a test fails -teardown() { - if [ "$BATS_TEST_COMPLETED" != 1 ]; then - echo "" >&2 - echo "========================================" >&2 - echo "TEST FAILED: ${BATS_TEST_NAME}" >&2 - echo "========================================" >&2 - - echo "" >&2 - echo "--- Exporter logs (test-exporter-oidc) ---" >&2 - if [ -f "$EXPORTER_LOGS_DIR/test-exporter-oidc.log" ]; then - tail -250 "$EXPORTER_LOGS_DIR/test-exporter-oidc.log" >&2 - else - echo "(no log file found)" >&2 - fi - - echo "" >&2 - echo "--- Exporter logs (test-exporter-sa) ---" >&2 - if [ -f "$EXPORTER_LOGS_DIR/test-exporter-sa.log" ]; then - tail -250 "$EXPORTER_LOGS_DIR/test-exporter-sa.log" >&2 - else - echo "(no log file found)" >&2 - fi - - echo "" >&2 - echo "--- Exporter logs (test-exporter-legacy) ---" >&2 - if [ -f "$EXPORTER_LOGS_DIR/test-exporter-legacy.log" ]; then - tail -250 "$EXPORTER_LOGS_DIR/test-exporter-legacy.log" >&2 - else - echo "(no log file found)" >&2 - fi - - echo "" >&2 - echo "--- Controller logs (last 250 lines) ---" >&2 - # operator uses component=controller, helm uses control-plane=controller-manager - kubectl -n "${JS_NAMESPACE}" logs -l component=controller --tail=250 2>&1 >&2 \ - || kubectl -n "${JS_NAMESPACE}" logs -l control-plane=controller-manager --tail=250 2>&1 >&2 || true - - echo "" >&2 - echo "--- Router logs (last 250 lines) ---" >&2 - # operator uses component=router, helm uses control-plane=controller-router - kubectl -n "${JS_NAMESPACE}" logs -l component=router --tail=250 2>&1 >&2 \ - || kubectl -n "${JS_NAMESPACE}" logs -l control-plane=controller-router --tail=250 2>&1 >&2 || true - - echo "========================================" >&2 - fi -} - -# teardown_file runs once after all tests complete (requires bats-core 1.5.0+) -teardown_file() { - echo "" >&2 - echo "========================================" >&2 - echo "TEARDOWN_FILE RUNNING" >&2 - echo "========================================" >&2 - echo "=== Cleaning up exporter bash processes ===" >&2 - - # Read PIDs from file - if [ -f "$EXPORTER_PIDS_FILE" ]; then - local pids=$(cat "$EXPORTER_PIDS_FILE" | tr '\n' ' ') - echo "Tracked PIDs from file: $pids" >&2 - - while IFS= read -r pid; do - if [ -n "$pid" ]; then - echo "Checking PID $pid..." >&2 - if ps -p "$pid" > /dev/null 2>&1; then - echo " Killing PID $pid" >&2 - kill -9 "$pid" 2>/dev/null || true - else - echo " PID $pid already terminated" >&2 - fi - fi - done < "$EXPORTER_PIDS_FILE" - else - echo "No PIDs file found at $EXPORTER_PIDS_FILE" >&2 - fi - - echo "Checking for orphaned jmp processes..." >&2 - local orphans=$(pgrep -f "jmp run --exporter" 2>/dev/null | wc -l) - echo "Found $orphans orphaned jmp processes" >&2 - - # remove orphaned processes - pkill -9 -f "jmp run --exporter" 2>/dev/null || true - - # Clean up the PIDs file - rm -f "$EXPORTER_PIDS_FILE" - - echo "=== Cleanup complete ===" >&2 -} - -wait_for_exporter() { - # After a lease operation the exporter disconnects from the controller and reconnects. - # The disconnect can take a short while so let's avoid catching the pre-disconnect state. - sleep 2 - - # Wait for Online + Registered conditions AND exporterStatus=Available. - # Online+Registered alone are never cleared between leases, so they can't detect - # whether the exporter has finished cleaning up a previous lease. Checking - # exporterStatus=Available ensures the exporter's serve() loop has fully processed - # the lease-end and is ready to accept new leases. (See #425) - _wait_for_single_exporter test-exporter-oidc & - local pid1=$! - _wait_for_single_exporter test-exporter-sa & - local pid2=$! - _wait_for_single_exporter test-exporter-legacy & - local pid3=$! - - # Wait for all to complete and capture failures - local rc=0 - wait "$pid1" || rc=$? - wait "$pid2" || rc=$? - wait "$pid3" || rc=$? - return $rc -} - -_wait_for_single_exporter() { - local name="$1" - local timeout=300 # 5 minutes - local elapsed=0 - - # First wait for the basic conditions (fast path for initial registration) - kubectl -n "${JS_NAMESPACE}" wait --timeout "${timeout}s" --for=condition=Online --for=condition=Registered \ - "exporters.jumpstarter.dev/${name}" - - # Then poll until exporterStatus is Available (not leased or cleaning up) - while [ $elapsed -lt $timeout ]; do - local status - status=$(kubectl -n "${JS_NAMESPACE}" get "exporters.jumpstarter.dev/${name}" \ - -o jsonpath='{.status.exporterStatus}' 2>/dev/null || echo "") - if [ "$status" = "Available" ]; then - return 0 - fi - sleep 0.5 - elapsed=$((elapsed + 1)) - done - - echo "Timed out waiting for ${name} to reach Available status" >&2 - return 1 -} - -@test "login endpoint serves landing page" { - # Check if login service exists - - run curl -s http://${LOGIN_ENDPOINT} - assert_success - - # Verify the response is HTML with login instructions - assert_output --partial "Jumpstarter" - assert_output --partial "jmp login" - -} - -@test "can create clients with admin cli" { - run jmp admin create client -n "${JS_NAMESPACE}" test-client-oidc --unsafe --nointeractive \ - --oidc-username dex:test-client-oidc - assert_success - - run jmp admin create client -n "${JS_NAMESPACE}" test-client-sa --unsafe --nointeractive \ - --oidc-username dex:system:serviceaccount:"${JS_NAMESPACE}":test-client-sa - assert_success - - run jmp admin create client -n "${JS_NAMESPACE}" test-client-legacy --unsafe --save - assert_success - - run jmp config client list -o yaml - assert_success - assert_output --partial "test-client-legacy" -} - -@test "can create exporters with admin cli" { - run jmp admin create exporter -n "${JS_NAMESPACE}" test-exporter-oidc --nointeractive \ - --oidc-username dex:test-exporter-oidc \ - --label example.com/board=oidc - assert_success - - run jmp admin create exporter -n "${JS_NAMESPACE}" test-exporter-sa --nointeractive \ - --oidc-username dex:system:serviceaccount:"${JS_NAMESPACE}":test-exporter-sa \ - --label example.com/board=sa - assert_success - - run jmp admin create exporter -n "${JS_NAMESPACE}" test-exporter-legacy --save \ - --label example.com/board=legacy - assert_success - - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/test-exporter-legacy.yaml - run jmp config exporter list -o yaml - assert_success - assert_output --partial "test-exporter-legacy" -} - -@test "can login with oidc test-client-oidc" { - run jmp login --client test-client-oidc \ - --endpoint "$ENDPOINT" --namespace "${JS_NAMESPACE}" --name test-client-oidc \ - --issuer https://dex.dex.svc.cluster.local:5556 \ - --username test-client-oidc@example.com --password password --unsafe - assert_success - run jmp config client list -o yaml - assert_success - assert_output --partial "test-client-oidc" -} - -@test "can login with oidc test-client-oidc-provisioning" { - run jmp login --client test-client-oidc-provisioning-example-com \ - --endpoint "$ENDPOINT" --namespace "${JS_NAMESPACE}" --name="" \ - --issuer https://dex.dex.svc.cluster.local:5556 \ - --username test-client-oidc-provisioning@example.com --password password --unsafe - assert_success - run jmp config client list -o yaml - assert_success - assert_output --partial "test-client-oidc-provisioning-example-com" -} - -@test "can login with oidc test-client-sa" { - run jmp login --client test-client-sa \ - --endpoint "$ENDPOINT" --namespace "${JS_NAMESPACE}" --name test-client-sa \ - --issuer https://dex.dex.svc.cluster.local:5556 \ - --connector-id kubernetes \ - --token $(kubectl create -n "${JS_NAMESPACE}" token test-client-sa) --unsafe - assert_success - run jmp config client list -o yaml - assert_success - assert_output --partial "test-client-sa" -} - -@test "can login with oidc test-exporter-oidc" { - run jmp login --exporter test-exporter-oidc --name test-exporter-oidc \ - --endpoint "$ENDPOINT" --namespace "${JS_NAMESPACE}" \ - --issuer https://dex.dex.svc.cluster.local:5556 \ - --username test-exporter-oidc@example.com --password password - assert_success - # add the mock export paths to those files - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/test-exporter-oidc.yaml - run jmp config exporter list -o yaml - assert_success - assert_output --partial "test-exporter-oidc" - -} - -@test "can login with oidc test-exporter-sa" { - run jmp login --exporter test-exporter-sa \ - --endpoint "$ENDPOINT" --namespace "${JS_NAMESPACE}" --name test-exporter-sa \ - --issuer https://dex.dex.svc.cluster.local:5556 \ - --connector-id kubernetes \ - --token $(kubectl create -n "${JS_NAMESPACE}" token test-exporter-sa) - assert_success - - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/test-exporter-oidc.yaml - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/test-exporter-sa.yaml - go run github.com/mikefarah/yq/v4@latest -i ". * load(\"e2e/exporters/exporter.yaml\")" \ - /etc/jumpstarter/exporters/test-exporter-legacy.yaml - - run jmp config exporter list -o yaml - assert_success - assert_output --partial "test-exporter-sa" -} - -@test "can login with simplified login" { - # This test only works with operator-based deployment, which deploys the CA ConfigMap - if [ "${METHOD:-}" != "operator" ]; then - skip "CA certificate injection only configured with operator deployment (METHOD=$METHOD)" - fi - - jmp config client delete test-client-oidc - - run jmp login test-client-oidc@http://${LOGIN_ENDPOINT} --insecure-tls --nointeractive \ - --username test-client-oidc@example.com --password password --unsafe - assert_success - - # Verify CA certificate is populated in client config - local client_config="${HOME}/.config/jumpstarter/clients/test-client-oidc.yaml" - run test -f "$client_config" - assert_success - run go run github.com/mikefarah/yq/v4@latest '.tls.ca' "$client_config" - assert_success - refute_output "" - refute_output "null" - echo "Client config has CA certificate populated" - - # Verify the new client is set as the default (marked with * in CURRENT column) - run jmp config client list - assert_success - assert_line --regexp '^[[:space:]]*\*[[:space:]]+test-client-oidc[[:space:]]' - echo "Client test-client-oidc is set as default" -} - -@test "can run exporters" { - cat <&- & -while true; do - jmp run --exporter test-exporter-oidc >> "$EXPORTER_LOGS_DIR/test-exporter-oidc.log" 2>&1 -done -EOF - echo "$!" >> "$EXPORTER_PIDS_FILE" - - cat <&- & -while true; do - jmp run --exporter test-exporter-sa >> "$EXPORTER_LOGS_DIR/test-exporter-sa.log" 2>&1 -done -EOF - echo "$!" >> "$EXPORTER_PIDS_FILE" - - cat <&- & -while true; do - jmp run --exporter test-exporter-legacy >> "$EXPORTER_LOGS_DIR/test-exporter-legacy.log" 2>&1 -done -EOF - echo "$!" >> "$EXPORTER_PIDS_FILE" - - wait_for_exporter -} - -@test "can specify client config only using environment variables" { - wait_for_exporter - - # we feed the namespace into JMP_NAMESPACE along with all the other client details - # to verify that the client can operate without a config file - JMP_NAMESPACE="${JS_NAMESPACE}" \ - JMP_DRIVERS_ALLOW="*" \ - JMP_NAME=test-client-legacy \ - JMP_ENDPOINT=$(kubectl get clients.jumpstarter.dev -n "${JS_NAMESPACE}" test-client-legacy -o 'jsonpath={.status.endpoint}') \ - JMP_TOKEN=$(kubectl get secrets -n "${JS_NAMESPACE}" test-client-legacy-client -o 'jsonpath={.data.token}' | base64 -d) \ - jmp shell --selector example.com/board=oidc j power on -} - -@test "legacy client config contains CA certificate and works with secure TLS" { - # This test only works with operator-based deployment, which creates the CA ConfigMap - if [ "${METHOD:-}" != "operator" ]; then - skip "CA certificate injection only available with operator deployment (METHOD=$METHOD)" - fi - - wait_for_exporter - - # Get the config file path from jmp (clients are saved to ~/.config/jumpstarter/clients/) - local config_file="${HOME}/.config/jumpstarter/clients/test-client-legacy.yaml" - run test -f "$config_file" - assert_success - - # Check that tls.ca field exists and is not empty - run go run github.com/mikefarah/yq/v4@latest '.tls.ca' "$config_file" - assert_success - # The CA should be a non-empty base64-encoded string - refute_output "" - refute_output "null" - - # Test that the client works WITHOUT JUMPSTARTER_GRPC_INSECURE set - # This proves the CA certificate is being used for TLS verification - run env -u JUMPSTARTER_GRPC_INSECURE jmp get exporters --client test-client-legacy -o yaml - assert_success - # Should see the legacy exporter in the output - assert_output --partial "test-exporter-legacy" -} - -@test "can operate on leases" { - wait_for_exporter - - jmp config client use test-client-oidc - - jmp create lease --selector example.com/board=oidc --duration 1d - jmp get leases - jmp get exporters - - # Verify label selector filtering works (regression test for issue #36) - run jmp get leases --selector example.com/board=oidc -o yaml - assert_success - assert_output --partial "example.com/board=oidc" - - run jmp get leases --selector example.com/board=doesnotexist - assert_success - assert_output "No resources found." - - # Test complex selectors with matchExpressions (regression test for server-side over-filtering) - # Use 'sa' exporter since 'oidc' is already leased above. The '!nonexistent' is a matchExpression - # that will always be true (label doesn't exist on exporters), allowing the lease to match. - jmp create lease --selector 'example.com/board=sa,!nonexistent' --duration 1d - - # Partial match: filter with just matchLabels (subset) → expecting a match - run jmp get leases --selector 'example.com/board=sa' -o yaml - assert_success - assert_output --partial "example.com/board=sa" - - # Partial match: filter with just matchExpressions (subset) → expecting a match - # This specifically tests client-side filtering of matchExpressions - run jmp get leases --selector '!nonexistent' -o yaml - assert_success - assert_output --partial "!nonexistent" - - # Non-matching matchExpressions → expecting no match with current implementation - # where we're filtering against the original lease request - run jmp get leases --selector 'example.com/board=sa,!production' - assert_success - assert_output "No resources found." - - # Filter asks for more than lease has → expecting no match - run jmp get leases --selector 'example.com/board=sa,!nonexistent,region=us' - assert_success - assert_output "No resources found." - - jmp delete leases --all -} - -@test "paginated lease listing returns all leases" { - wait_for_exporter - - jmp config client use test-client-oidc - - for i in $(seq 1 101); do - run jmp create lease --selector example.com/board=oidc --duration 1d - assert_success - done - - run jmp get leases -o name - assert_success - [ "$(echo "$output" | wc -l)" -eq 101 ] - - jmp delete leases --all -} - -@test "paginated exporter listing returns all exporters" { - wait_for_exporter - - jmp config client use test-client-oidc - - for i in $(seq 1 101); do - run jmp admin create exporter -n "${JS_NAMESPACE}" "pagination-exp-${i}" --nointeractive \ - -l pagination=true --oidc-username "dex:pagination-exp-${i}" - assert_success - done - - run jmp get exporters --selector pagination=true -o name - assert_success - [ "$(echo "$output" | wc -l)" -eq 101 ] - - for i in $(seq 1 101); do - jmp admin delete exporter --namespace "${JS_NAMESPACE}" "pagination-exp-${i}" --delete - done -} - -@test "lease listing shows expires at and remaining columns" { - wait_for_exporter - - jmp config client use test-client-oidc - - jmp create lease --selector example.com/board=oidc --duration 1d - - run env COLUMNS=200 jmp get leases - assert_success - assert_output --partial "EXPIRES AT" - assert_output --partial "REMAINING" - jmp delete leases --all -} - -@test "can transfer lease to another client" { - wait_for_exporter - - jmp config client use test-client-oidc - - # Create a lease owned by test-client-oidc - run jmp create lease --selector example.com/board=oidc --duration 1d -o yaml - assert_success - LEASE_NAME=$(echo "$output" | go run github.com/mikefarah/yq/v4@latest '.name') - - # Wait for the lease to become active - kubectl -n "${JS_NAMESPACE}" wait --timeout 60s --for=condition=Ready \ - leases.jumpstarter.dev/"$LEASE_NAME" - - # Transfer the lease to test-client-legacy - run jmp update lease "$LEASE_NAME" --to-client test-client-legacy -o yaml - assert_success - assert_output --partial "test-client-legacy" - - # Delete as the new owner - jmp delete leases --client test-client-legacy --all -} - -@test "can lease and connect to exporters" { - wait_for_exporter - - jmp shell --client test-client-oidc --selector example.com/board=oidc j power on - jmp shell --client test-client-sa --selector example.com/board=sa j power on - jmp shell --client test-client-legacy --selector example.com/board=legacy j power on - - wait_for_exporter - jmp shell --client test-client-oidc-provisioning-example-com --selector example.com/board=oidc j power on -} - -@test "can lease and connect to exporters by name" { - wait_for_exporter - - jmp shell --client test-client-oidc --name test-exporter-oidc j power on - jmp shell --client test-client-sa --name test-exporter-sa j power on - jmp shell --client test-client-legacy --name test-exporter-legacy j power on - - # Reusing the same exporter immediately can be flaky while it reconnects. - wait_for_exporter - - # --name and --selector together should work when they match. - jmp shell --client test-client-oidc --name test-exporter-oidc --selector example.com/board=oidc j power on -} - -@test "fails fast when requesting non-existent exporter by name" { - wait_for_exporter - - # Strict behavior: missing named exporter should become Unsatisfiable and fail quickly. - # If controller returns Pending here, this command will likely hit timeout (exit 124). - run timeout 20s jmp shell --client test-client-oidc --name test-exporter-does-not-exist j power on - assert_failure - [ "$status" -ne 124 ] - assert_output --partial "cannot be satisfied" -} - -@test "can get crds with admin cli" { - jmp admin get client --namespace "${JS_NAMESPACE}" - jmp admin get exporter --namespace "${JS_NAMESPACE}" - jmp admin get lease --namespace "${JS_NAMESPACE}" -} - -@test "can delete clients with admin cli" { - kubectl -n "${JS_NAMESPACE}" get secret test-client-oidc-client - kubectl -n "${JS_NAMESPACE}" get clients.jumpstarter.dev/test-client-oidc - kubectl -n "${JS_NAMESPACE}" get clients.jumpstarter.dev/test-client-sa - kubectl -n "${JS_NAMESPACE}" get clients.jumpstarter.dev/test-client-legacy - - jmp admin delete client --namespace "${JS_NAMESPACE}" test-client-oidc --delete - jmp admin delete client --namespace "${JS_NAMESPACE}" test-client-sa --delete - jmp admin delete client --namespace "${JS_NAMESPACE}" test-client-legacy --delete - - run ! kubectl -n "${JS_NAMESPACE}" get secret test-client-oidc-client - run ! kubectl -n "${JS_NAMESPACE}" get clients.jumpstarter.dev/test-client-oidc - run ! kubectl -n "${JS_NAMESPACE}" get clients.jumpstarter.dev/test-client-sa - run ! kubectl -n "${JS_NAMESPACE}" get clients.jumpstarter.dev/test-client-legacy -} - -@test "can delete exporters with admin cli" { - kubectl -n "${JS_NAMESPACE}" get secret test-exporter-oidc-exporter - kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-oidc - kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-sa - kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-legacy - - jmp admin delete exporter --namespace "${JS_NAMESPACE}" test-exporter-oidc --delete - jmp admin delete exporter --namespace "${JS_NAMESPACE}" test-exporter-sa --delete - jmp admin delete exporter --namespace "${JS_NAMESPACE}" test-exporter-legacy --delete - - run ! kubectl -n "${JS_NAMESPACE}" get secret test-exporter-oidc-exporter - run ! kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-oidc - run ! kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-sa - run ! kubectl -n "${JS_NAMESPACE}" get exporters.jumpstarter.dev/test-exporter-legacy -}