From 18ee887f78c425fd699f73c6af279f9f0d591086 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 11 Feb 2026 11:30:20 +0800 Subject: [PATCH 1/6] CI: refactor CI workflow for cloudberry-backup - replace the previous single-job flow with a two-stage build/test pipeline - build Cloudberry from source and publish the install as an artifact - run unit/integration/end_to_end tests in parallel jobs - add Apache header and CI documentation in the workflow - capture logs and upload artifacts on failure with a test summary --- .github/workflows/cloudberry-backup-ci.yml | 312 +++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 .github/workflows/cloudberry-backup-ci.yml diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml new file mode 100644 index 00000000..4e7c3267 --- /dev/null +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -0,0 +1,312 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# GitHub Actions Workflow: Cloudberry-Backup CI +# -------------------------------------------------------------------- +# Description: +# +# Builds Apache Cloudberry from source, packages the installation, +# and runs Cloudberry-Backup tests against a demo Cloudberry cluster. +# +# Workflow Overview: +# 1. Build Cloudberry from source and upload the installation as an artifact. +# 2. For each test target (unit, integration, end_to_end), restore the +# Cloudberry installation, create a demo cluster, and run tests. +# +# Notes: +# - Each test job runs in an isolated environment and creates its own demo cluster. +# - Artifacts are used to avoid rebuilding Cloudberry for every test target. +# -------------------------------------------------------------------- + +name: cloudberry-backup-ci + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [ opened, synchronize, reopened, edited, ready_for_review ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + CLOUDBERRY_REPO: apache/cloudberry + CLOUDBERRY_REF: main + CLOUDBERRY_DIR: cloudberry + CLOUDBERRY_BACKUP_DIR: cloudberry-backup + +jobs: + build_cloudberry: + name: Build Cloudberry From Source + runs-on: ubuntu-22.04 + timeout-minutes: 180 + container: + image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + run: | + set -euo pipefail + echo "=== Disk space before cleanup ===" + df -h / + + rm -rf /host_opt/hostedtoolcache || true + rm -rf /host_usr_local/lib/android || true + rm -rf /host_usr_share/dotnet || true + rm -rf /host_opt/ghc || true + rm -rf /host_usr_local/.ghcup || true + rm -rf /host_usr_share/swift || true + rm -rf /host_usr_local/share/powershell || true + rm -rf /host_usr_local/share/chromium || true + rm -rf /host_usr_share/miniconda || true + rm -rf /host_opt/az || true + rm -rf /host_usr_share/sbt || true + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Checkout Cloudberry Source + uses: actions/checkout@v4 + with: + repository: ${{ env.CLOUDBERRY_REPO }} + ref: ${{ env.CLOUDBERRY_REF }} + fetch-depth: 1 + submodules: true + path: ${{ env.CLOUDBERRY_DIR }} + + - name: Cloudberry Environment Initialization + run: | + set -euo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + chown -R gpadmin:gpadmin . + chmod -R 755 . + df -h / + rm -rf /__t/* + df -h / + + - name: Configure Cloudberry + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + run: | + set -euo pipefail + chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh" + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure script failed" + exit 1 + fi + + - name: Build Cloudberry + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + run: | + set -euo pipefail + chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh" + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build script failed" + exit 1 + fi + + - name: Package Cloudberry Installation + run: | + set -euo pipefail + tar -C /usr/local -czf cloudberry-db.tgz cloudberry-db + + - name: Upload Cloudberry Installation + uses: actions/upload-artifact@v4 + with: + name: cloudberry-db-install + path: cloudberry-db.tgz + if-no-files-found: error + retention-days: 7 + + test_cloudberry_backup: + name: Run Cloudberry-Backup Tests (${{ matrix.test_target }}) + needs: [build_cloudberry] + runs-on: ubuntu-22.04 + timeout-minutes: 180 + container: + image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + strategy: + fail-fast: false + matrix: + test_target: [unit, integration, end_to_end] + + steps: + - name: Free Disk Space + run: | + set -euo pipefail + echo "=== Disk space before cleanup ===" + df -h / + + rm -rf /host_opt/hostedtoolcache || true + rm -rf /host_usr_local/lib/android || true + rm -rf /host_usr_share/dotnet || true + rm -rf /host_opt/ghc || true + rm -rf /host_usr_local/.ghcup || true + rm -rf /host_usr_share/swift || true + rm -rf /host_usr_local/share/powershell || true + rm -rf /host_usr_local/share/chromium || true + rm -rf /host_usr_share/miniconda || true + rm -rf /host_opt/az || true + rm -rf /host_usr_share/sbt || true + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Checkout Cloudberry-Backup + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: ${{ env.CLOUDBERRY_BACKUP_DIR }} + + - name: Checkout Cloudberry Source + uses: actions/checkout@v4 + with: + repository: ${{ env.CLOUDBERRY_REPO }} + ref: ${{ env.CLOUDBERRY_REF }} + fetch-depth: 1 + submodules: true + path: ${{ env.CLOUDBERRY_DIR }} + + - name: Download Cloudberry Installation + uses: actions/download-artifact@v4 + with: + name: cloudberry-db-install + + - name: Restore Cloudberry Installation + run: | + set -euo pipefail + tar -C /usr/local -xzf cloudberry-db.tgz + + - name: Cloudberry Environment Initialization + run: | + set -euo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + chown -R gpadmin:gpadmin . + chmod -R 755 . + df -h / + rm -rf /__t/* + df -h / + + - name: Create Cloudberry Demo Cluster + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + run: | + set -euo pipefail + chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh" + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + + - name: Run Cloudberry-Backup Tests + env: + CLOUDBERRY_BACKUP_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_BACKUP_DIR }} + CLOUDBERRY_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + TEST_TARGET: ${{ matrix.test_target }} + run: | + set -euo pipefail + TEST_LOG_ROOT="${GITHUB_WORKSPACE}/test-logs/${TEST_TARGET}" + mkdir -p "${TEST_LOG_ROOT}" + chown -R gpadmin:gpadmin "${TEST_LOG_ROOT}" + + cat <<'SCRIPT' > /tmp/run_cloudberry_backup_tests.sh + #!/bin/bash + set -euo pipefail + + export GOPATH=/home/gpadmin/go + export PATH=/usr/local/go/bin:${GOPATH}/bin:${PATH} + + source /usr/local/cloudberry-db/cloudberry-env.sh + source ${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh + + mkdir -p ${GOPATH}/src/github.com/apache + rsync -a --delete ${CLOUDBERRY_BACKUP_SRC}/ ${GOPATH}/src/github.com/apache/cloudberry-backup/ + + pushd ${GOPATH}/src/github.com/apache/cloudberry-backup + make depend + + out=$(psql postgres -c "select version();") + export TEST_GPDB_VERSION=$(echo "$out" | sed -n 's/.*Cloudberry Database \([0-9].[0-9]\+.[0-9]\+\).*/\1/p') + + make "${TEST_TARGET}" 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-${TEST_TARGET}.log" + popd + + if [ -n "${MASTER_DATA_DIRECTORY:-}" ] && [ -d "${MASTER_DATA_DIRECTORY}/log" ]; then + cp -a "${MASTER_DATA_DIRECTORY}/log" "${TEST_LOG_ROOT}/gpdb-log" || true + fi + if [ -d "${CLOUDBERRY_SRC}/build-logs" ]; then + cp -a "${CLOUDBERRY_SRC}/build-logs" "${TEST_LOG_ROOT}/cloudberry-build-logs" || true + fi + SCRIPT + + chmod +x /tmp/run_cloudberry_backup_tests.sh + set +e + su - gpadmin -c "TEST_LOG_ROOT=${TEST_LOG_ROOT} CLOUDBERRY_BACKUP_SRC=${CLOUDBERRY_BACKUP_SRC} CLOUDBERRY_SRC=${CLOUDBERRY_SRC} TEST_TARGET=${TEST_TARGET} /tmp/run_cloudberry_backup_tests.sh" + status=$? + set -e + + { + echo "## Cloudberry-Backup Test Summary" + echo "- Target: ${TEST_TARGET}" + if [ ${status} -eq 0 ]; then + echo "- Result: PASS" + else + echo "- Result: FAIL" + fi + echo "- Logs: ${TEST_LOG_ROOT}" + } >> "$GITHUB_STEP_SUMMARY" + + exit ${status} + + - name: Upload Test Logs (On Failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cloudberry-backup-logs-${{ matrix.test_target }} + path: test-logs/${{ matrix.test_target }} + if-no-files-found: warn + retention-days: 7 From 0619c59e13b6246b71b112d673c5f496102392c8 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 11 Feb 2026 12:11:37 +0800 Subject: [PATCH 2/6] Update Cloudberry CI workflow --- .github/workflows/cloudberry-backup-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml index 4e7c3267..464738c7 100644 --- a/.github/workflows/cloudberry-backup-ci.yml +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -103,6 +103,8 @@ jobs: path: ${{ env.CLOUDBERRY_DIR }} - name: Cloudberry Environment Initialization + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} run: | set -euo pipefail if ! su - gpadmin -c "/tmp/init_system.sh"; then @@ -110,8 +112,10 @@ jobs: exit 1 fi + mkdir -p "${SRC_DIR}/build-logs/details" chown -R gpadmin:gpadmin . chmod -R 755 . + chmod 777 "${SRC_DIR}/build-logs" df -h / rm -rf /__t/* df -h / @@ -218,6 +222,8 @@ jobs: tar -C /usr/local -xzf cloudberry-db.tgz - name: Cloudberry Environment Initialization + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} run: | set -euo pipefail if ! su - gpadmin -c "/tmp/init_system.sh"; then @@ -225,8 +231,10 @@ jobs: exit 1 fi + mkdir -p "${SRC_DIR}/build-logs/details" chown -R gpadmin:gpadmin . chmod -R 755 . + chmod 777 "${SRC_DIR}/build-logs" df -h / rm -rf /__t/* df -h / @@ -272,7 +280,10 @@ jobs: out=$(psql postgres -c "select version();") export TEST_GPDB_VERSION=$(echo "$out" | sed -n 's/.*Cloudberry Database \([0-9].[0-9]\+.[0-9]\+\).*/\1/p') + set +e make "${TEST_TARGET}" 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-${TEST_TARGET}.log" + test_status=${PIPESTATUS[0]} + set -e popd if [ -n "${MASTER_DATA_DIRECTORY:-}" ] && [ -d "${MASTER_DATA_DIRECTORY}/log" ]; then @@ -281,6 +292,8 @@ jobs: if [ -d "${CLOUDBERRY_SRC}/build-logs" ]; then cp -a "${CLOUDBERRY_SRC}/build-logs" "${TEST_LOG_ROOT}/cloudberry-build-logs" || true fi + + exit ${test_status} SCRIPT chmod +x /tmp/run_cloudberry_backup_tests.sh From 18367e260fd8767b8612fd1b4a9fe0f097b2fab4 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 11 Feb 2026 12:33:16 +0800 Subject: [PATCH 3/6] Align CI with Cloudberry build --- .github/workflows/cloudberry-backup-ci.yml | 46 +++++++++++++++------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml index 464738c7..e7095458 100644 --- a/.github/workflows/cloudberry-backup-ci.yml +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -17,12 +17,12 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- -# GitHub Actions Workflow: Cloudberry-Backup CI +# GitHub Actions Workflow: Cloudberry-backup CI # -------------------------------------------------------------------- # Description: # # Builds Apache Cloudberry from source, packages the installation, -# and runs Cloudberry-Backup tests against a demo Cloudberry cluster. +# and runs Cloudberry-backup tests against a demo Cloudberry cluster. # # Workflow Overview: # 1. Build Cloudberry from source and upload the installation as an artifact. @@ -142,6 +142,13 @@ jobs: exit 1 fi + - name: Package Cloudberry Source + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + run: | + set -euo pipefail + tar -C "${GITHUB_WORKSPACE}" -czf cloudberry-src.tgz "${CLOUDBERRY_DIR}" + - name: Package Cloudberry Installation run: | set -euo pipefail @@ -155,8 +162,16 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Upload Cloudberry Source + uses: actions/upload-artifact@v4 + with: + name: cloudberry-source + path: cloudberry-src.tgz + if-no-files-found: error + retention-days: 7 + test_cloudberry_backup: - name: Run Cloudberry-Backup Tests (${{ matrix.test_target }}) + name: Cloudberry-backup Tests (${{ matrix.test_target }}) needs: [build_cloudberry] runs-on: ubuntu-22.04 timeout-minutes: 180 @@ -196,31 +211,32 @@ jobs: echo "=== Disk space after cleanup ===" df -h / - - name: Checkout Cloudberry-Backup + - name: Checkout Cloudberry-backup uses: actions/checkout@v4 with: fetch-depth: 0 path: ${{ env.CLOUDBERRY_BACKUP_DIR }} - - name: Checkout Cloudberry Source - uses: actions/checkout@v4 - with: - repository: ${{ env.CLOUDBERRY_REPO }} - ref: ${{ env.CLOUDBERRY_REF }} - fetch-depth: 1 - submodules: true - path: ${{ env.CLOUDBERRY_DIR }} - - name: Download Cloudberry Installation uses: actions/download-artifact@v4 with: name: cloudberry-db-install + - name: Download Cloudberry Source + uses: actions/download-artifact@v4 + with: + name: cloudberry-source + - name: Restore Cloudberry Installation run: | set -euo pipefail tar -C /usr/local -xzf cloudberry-db.tgz + - name: Restore Cloudberry Source + run: | + set -euo pipefail + tar -C "${GITHUB_WORKSPACE}" -xzf cloudberry-src.tgz + - name: Cloudberry Environment Initialization env: SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} @@ -250,7 +266,7 @@ jobs: exit 1 fi - - name: Run Cloudberry-Backup Tests + - name: Cloudberry-backup Tests env: CLOUDBERRY_BACKUP_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_BACKUP_DIR }} CLOUDBERRY_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} @@ -303,7 +319,7 @@ jobs: set -e { - echo "## Cloudberry-Backup Test Summary" + echo "## Cloudberry-backup Test Summary" echo "- Target: ${TEST_TARGET}" if [ ${status} -eq 0 ]; then echo "- Result: PASS" From 49129e3ffa36ceedb09fa3cab57075aa1fa6a850 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 11 Feb 2026 15:05:36 +0800 Subject: [PATCH 4/6] Adjust CI setup for Cloudberry tests --- .github/workflows/cloudberry-backup-ci.yml | 75 +++++++++++++++++++++- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml index e7095458..e1aa045a 100644 --- a/.github/workflows/cloudberry-backup-ci.yml +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -293,11 +293,36 @@ jobs: pushd ${GOPATH}/src/github.com/apache/cloudberry-backup make depend - out=$(psql postgres -c "select version();") - export TEST_GPDB_VERSION=$(echo "$out" | sed -n 's/.*Cloudberry Database \([0-9].[0-9]\+.[0-9]\+\).*/\1/p') + dummy_dir=$(find ${CLOUDBERRY_SRC} -name dummy_seclabel -type d | head -n 1 || true) + if [ -n "${dummy_dir}" ]; then + pushd "${dummy_dir}" + make install + popd + gpconfig -c shared_preload_libraries -v dummy_seclabel + gpstop -ra + gpconfig -s shared_preload_libraries | grep dummy_seclabel + else + echo "dummy_seclabel not found, skipping preload setup" + fi + + psql postgres -c 'DROP TABLESPACE IF EXISTS test_tablespace' set +e - make "${TEST_TARGET}" 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-${TEST_TARGET}.log" + case "${TEST_TARGET}" in + unit) + make unit 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-unit.log" + ;; + integration) + make integration 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-integration.log" + ;; + end_to_end) + make end_to_end 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-end_to_end.log" + ;; + *) + echo "unknown test target: ${TEST_TARGET}" + exit 2 + ;; + esac test_status=${PIPESTATUS[0]} set -e popd @@ -339,3 +364,47 @@ jobs: path: test-logs/${{ matrix.test_target }} if-no-files-found: warn retention-days: 7 + + test_s3_local: + name: Cloudberry-backup Tests (test-s3-local) + needs: [build_cloudberry] + runs-on: ubuntu-22.04 + timeout-minutes: 60 + + steps: + - name: Checkout Cloudberry-backup + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: ${{ env.CLOUDBERRY_BACKUP_DIR }} + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.21' + + - name: Run test-s3-local + env: + CLOUDBERRY_BACKUP_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_BACKUP_DIR }} + run: | + set -euo pipefail + + export GOPATH="${GITHUB_WORKSPACE}/go" + export PATH="${GOPATH}/bin:${PATH}" + + mkdir -p "${GOPATH}/src/github.com/apache" + rsync -a --delete "${CLOUDBERRY_BACKUP_SRC}/" "${GOPATH}/src/github.com/apache/cloudberry-backup/" + + pushd "${GOPATH}/src/github.com/apache/cloudberry-backup" + make depend + make test-s3-local 2>&1 | tee "${GITHUB_WORKSPACE}/test-s3-local.log" + popd + + - name: Upload test-s3-local logs (On Failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cloudberry-backup-logs-test-s3-local + path: test-s3-local.log + if-no-files-found: warn + retention-days: 7 From 9414a08c8bccef8109cd986d121754258f634f9b Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 11 Feb 2026 15:49:03 +0800 Subject: [PATCH 5/6] Prevent CI install to /bin --- .github/workflows/cloudberry-backup-ci.yml | 217 ++++++++++++++++++++- 1 file changed, 207 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml index e1aa045a..64353a03 100644 --- a/.github/workflows/cloudberry-backup-ci.yml +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -287,10 +287,7 @@ jobs: source /usr/local/cloudberry-db/cloudberry-env.sh source ${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh - mkdir -p ${GOPATH}/src/github.com/apache - rsync -a --delete ${CLOUDBERRY_BACKUP_SRC}/ ${GOPATH}/src/github.com/apache/cloudberry-backup/ - - pushd ${GOPATH}/src/github.com/apache/cloudberry-backup + pushd ${CLOUDBERRY_BACKUP_SRC} make depend dummy_dir=$(find ${CLOUDBERRY_SRC} -name dummy_seclabel -type d | head -n 1 || true) @@ -313,6 +310,7 @@ jobs: make unit 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-unit.log" ;; integration) + sudo dnf install glibc-langpack-de -y make integration 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-integration.log" ;; end_to_end) @@ -390,14 +388,19 @@ jobs: set -euo pipefail export GOPATH="${GITHUB_WORKSPACE}/go" - export PATH="${GOPATH}/bin:${PATH}" - - mkdir -p "${GOPATH}/src/github.com/apache" - rsync -a --delete "${CLOUDBERRY_BACKUP_SRC}/" "${GOPATH}/src/github.com/apache/cloudberry-backup/" + export PATH="${GOPATH}/bin:/usr/local/go/bin:${PATH}" - pushd "${GOPATH}/src/github.com/apache/cloudberry-backup" + pushd "${CLOUDBERRY_BACKUP_SRC}" make depend - make test-s3-local 2>&1 | tee "${GITHUB_WORKSPACE}/test-s3-local.log" + make build + + ${PWD}/plugins/generate_minio_config.sh + mkdir -p /tmp/minio/gpbackup-s3-test + docker run -d --name s3-minio --memory="2g" -p 9000:9000 -p 9001:9001 -v /tmp/minio:/data/minio quay.io/minio/minio server /data/minio --console-address ":9001" + sleep 2 + ${PWD}/plugins/plugin_test.sh "${GOPATH}/bin/gpbackup_s3_plugin" /tmp/minio_config.yaml 2>&1 | tee "${GITHUB_WORKSPACE}/test-s3-local.log" + docker stop s3-minio + docker rm s3-minio popd - name: Upload test-s3-local logs (On Failure) @@ -408,3 +411,197 @@ jobs: path: test-s3-local.log if-no-files-found: warn retention-days: 7 + + test_regression: + name: Cloudberry-backup Tests (regression) + needs: [build_cloudberry] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + container: + image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + run: | + set -euo pipefail + echo "=== Disk space before cleanup ===" + df -h / + + rm -rf /host_opt/hostedtoolcache || true + rm -rf /host_usr_local/lib/android || true + rm -rf /host_usr_share/dotnet || true + rm -rf /host_opt/ghc || true + rm -rf /host_usr_local/.ghcup || true + rm -rf /host_usr_share/swift || true + rm -rf /host_usr_local/share/powershell || true + rm -rf /host_usr_local/share/chromium || true + rm -rf /host_usr_share/miniconda || true + rm -rf /host_opt/az || true + rm -rf /host_usr_share/sbt || true + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Checkout Cloudberry-backup + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: ${{ env.CLOUDBERRY_BACKUP_DIR }} + + - name: Download Cloudberry Installation + uses: actions/download-artifact@v4 + with: + name: cloudberry-db-install + + - name: Download Cloudberry Source + uses: actions/download-artifact@v4 + with: + name: cloudberry-source + + - name: Restore Cloudberry Installation + run: | + set -euo pipefail + tar -C /usr/local -xzf cloudberry-db.tgz + + - name: Restore Cloudberry Source + run: | + set -euo pipefail + tar -C "${GITHUB_WORKSPACE}" -xzf cloudberry-src.tgz + + - name: Cloudberry Environment Initialization + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + run: | + set -euo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${SRC_DIR}/build-logs/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${SRC_DIR}/build-logs" + df -h / + rm -rf /__t/* + df -h / + + - name: Create Cloudberry Demo Cluster + env: + SRC_DIR: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + run: | + set -euo pipefail + chmod +x "${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh" + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + + - name: Cloudberry-backup Regression Test + env: + CLOUDBERRY_BACKUP_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_BACKUP_DIR }} + CLOUDBERRY_SRC: ${{ github.workspace }}/${{ env.CLOUDBERRY_DIR }} + run: | + set -euo pipefail + TEST_LOG_ROOT="${GITHUB_WORKSPACE}/test-logs/regression" + mkdir -p "${TEST_LOG_ROOT}" + chown -R gpadmin:gpadmin "${TEST_LOG_ROOT}" + + cat <<'SCRIPT' > /tmp/run_cloudberry_backup_regression.sh + #!/bin/bash + set -euo pipefail + + export GOPATH=/home/gpadmin/go + export PATH=/usr/local/go/bin:${GOPATH}/bin:${PATH} + + source /usr/local/cloudberry-db/cloudberry-env.sh + source ${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh + + pushd ${CLOUDBERRY_BACKUP_SRC} + make depend + make build + make install + popd + + pushd ${CLOUDBERRY_SRC}/src/test/regress + ./pg_regress --dbname=regression --host=localhost --port=7000 --init-file=init_file --schedule=./minimal_schedule || true + cat regression.diffs 2>/dev/null || true + popd + + psql -d postgres -c 'DROP TABLESPACE IF EXISTS test_tablespace' + + pg_dump regression -f /tmp/regression_schema_before.sql --schema-only + + backup_dir=/tmp/regression_backup + rm -rf "${backup_dir}" + mkdir -p "${backup_dir}" + + gpbackup --dbname regression --backup-dir "${backup_dir}" --metadata-only + + find "${backup_dir}" -name "*_metadata.sql" | sed -nr 's#.*gpseg-[0-9]/backups/.*/([0-9]{14})/.*#\\1#p' > /tmp/regression_timestamp + read timestamp < /tmp/regression_timestamp + echo "backup timestamp: ${timestamp}" + + psql -d postgres -c 'DROP DATABASE IF EXISTS regression' + + set +e + gprestore --create-db --timestamp ${timestamp} --backup-dir "${backup_dir}" --with-globals --on-error-continue + set -e + + pg_dump regression -f /tmp/regression_schema_after.sql --schema-only + + set +e + diff -u /tmp/regression_schema_before.sql /tmp/regression_schema_after.sql > /tmp/regression_schema.diff + diff_status=$? + set -e + + cp -a /tmp/regression_schema_before.sql "${TEST_LOG_ROOT}/regression_schema_before.sql" || true + cp -a /tmp/regression_schema_after.sql "${TEST_LOG_ROOT}/regression_schema_after.sql" || true + cp -a /tmp/regression_schema.diff "${TEST_LOG_ROOT}/regression_schema.diff" || true + + if [ -n "${MASTER_DATA_DIRECTORY:-}" ] && [ -d "${MASTER_DATA_DIRECTORY}/log" ]; then + cp -a "${MASTER_DATA_DIRECTORY}/log" "${TEST_LOG_ROOT}/gpdb-log" || true + fi + if [ -d "${CLOUDBERRY_SRC}/build-logs" ]; then + cp -a "${CLOUDBERRY_SRC}/build-logs" "${TEST_LOG_ROOT}/cloudberry-build-logs" || true + fi + + if [ ${diff_status} -ne 0 ]; then + echo "regression schema diff found" + exit ${diff_status} + fi + SCRIPT + + chmod +x /tmp/run_cloudberry_backup_regression.sh + set +e + su - gpadmin -c "TEST_LOG_ROOT=${TEST_LOG_ROOT} CLOUDBERRY_BACKUP_SRC=${CLOUDBERRY_BACKUP_SRC} CLOUDBERRY_SRC=${CLOUDBERRY_SRC} /tmp/run_cloudberry_backup_regression.sh" + status=$? + set -e + + { + echo "## Cloudberry-backup Test Summary" + echo "- Target: regression" + if [ ${status} -eq 0 ]; then + echo "- Result: PASS" + else + echo "- Result: FAIL" + fi + echo "- Logs: ${TEST_LOG_ROOT}" + } >> "$GITHUB_STEP_SUMMARY" + + exit ${status} + + - name: Upload Regression Logs (On Failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cloudberry-backup-logs-regression + path: test-logs/regression + if-no-files-found: warn + retention-days: 7 From 6a8bc2d78edf7dd6c413a874d38355964dabec89 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 11 Feb 2026 16:46:34 +0800 Subject: [PATCH 6/6] Remove old CI workflow --- .github/workflows/build_and_unit_test.yml | 44 ----------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/workflows/build_and_unit_test.yml diff --git a/.github/workflows/build_and_unit_test.yml b/.github/workflows/build_and_unit_test.yml deleted file mode 100644 index 9727fc74..00000000 --- a/.github/workflows/build_and_unit_test.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: build_and_unit_test - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - types: [ opened, synchronize, reopened, edited, ready_for_review ] - workflow_dispatch: - -jobs: - - build_and_unit_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - path: go/src/github.com/apache/cloudberry-backup - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.21 - - - name: Set Environment - run: | - echo "GOPATH=${GITHUB_WORKSPACE}/go" >> $GITHUB_ENV - echo "${GITHUB_WORKSPACE}/go/bin" >> $GITHUB_PATH - - - name: Dependencies - run: | - cd ${GOPATH}/src/github.com/apache/cloudberry-backup - make depend - - - name: Build - run: | - cd ${GOPATH}/src/github.com/apache/cloudberry-backup - make build - - - name: Unit Test - run: | - cd ${GOPATH}/src/github.com/apache/cloudberry-backup - make unit_all_gpdb_versions