From b3724be48d8a84f88158d54daa6c26d4804527c6 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 22 Mar 2026 19:15:55 +0800 Subject: [PATCH 1/7] refactor(workflows): consolidate pd-store-server publish flow - add dedicated reusable workflow for latest/release pd-store-server publishing - split multi-arch publishing into amd64 first, arm64 second, then manifest merge - introduce latest schedule and hash gate using LAST_SERVER_HASH - remove standalone server latest/release workflows and update README map --- .../_publish_pd_store_server_reusable.yml | 474 ++++++++++++++++++ .../publish_latest_pd_store_server_image.yml | 338 +------------ .../workflows/publish_latest_server_image.yml | 42 -- .../publish_release_pd_store_server_image.yml | 39 ++ .../publish_release_server_image.yml | 39 -- README.md | 55 +- 6 files changed, 561 insertions(+), 426 deletions(-) create mode 100644 .github/workflows/_publish_pd_store_server_reusable.yml delete mode 100644 .github/workflows/publish_latest_server_image.yml create mode 100644 .github/workflows/publish_release_pd_store_server_image.yml delete mode 100644 .github/workflows/publish_release_server_image.yml diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml new file mode 100644 index 0000000..bb52942 --- /dev/null +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -0,0 +1,474 @@ +name: "Publish pd-store-server image (reusable)" + +on: + workflow_call: + inputs: + mode: + description: "publish mode: latest or release" + required: true + type: string + repository_url: + description: "source repository in owner/name format" + required: false + default: apache/hugegraph + type: string + branch: + description: "source branch name" + required: true + type: string + mvn_args: + description: "mvn build args" + required: false + default: '' + type: string + strict_mode: + description: "whether integration precheck is mandatory before publish" + required: false + default: true + type: boolean + wait_timeout_sec: + description: "docker compose wait timeout in seconds" + required: false + default: '300' + type: string + enable_hash_gate: + description: "whether to skip latest publish if source hash unchanged" + required: false + default: false + type: boolean + last_hash_value: + description: "last published source hash" + required: false + default: '' + type: string + last_hash_name: + description: "repo variable name for latest hash" + required: false + default: '' + type: string + hash_repo_owner: + description: "owner of repo storing LAST_* hash variable" + required: false + default: hugegraph + type: string + hash_repo_name: + description: "repo name storing LAST_* hash variable" + required: false + default: actions + type: string + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_PASSWORD: + required: true + PERSONAL_ACCESS_TOKEN: + required: false + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + need_update: ${{ steps.prepare.outputs.need_update }} + source_sha: ${{ steps.prepare.outputs.source_sha }} + checkout_ref: ${{ steps.prepare.outputs.checkout_ref }} + version_tag: ${{ steps.prepare.outputs.version_tag }} + build_matrix_json: ${{ steps.prepare.outputs.build_matrix_json }} + steps: + - name: Resolve mode and source ref + id: prepare + env: + MODE: ${{ inputs.mode }} + REPOSITORY_URL: ${{ inputs.repository_url }} + BRANCH: ${{ inputs.branch }} + ENABLE_HASH_GATE: ${{ inputs.enable_hash_gate }} + LAST_HASH_VALUE: ${{ inputs.last_hash_value }} + run: | + set -euo pipefail + + if [ "$MODE" != "latest" ] && [ "$MODE" != "release" ]; then + echo "Invalid mode: $MODE. Expected latest or release." + exit 1 + fi + + source_sha="$(git ls-remote "https://github.com/${REPOSITORY_URL}.git" "refs/heads/${BRANCH}" | awk '{print $1}')" + if [ -z "$source_sha" ]; then + echo "Failed to resolve source SHA for ${REPOSITORY_URL}@${BRANCH}" + exit 1 + fi + + checkout_ref="$source_sha" + + if [ "$MODE" = "latest" ]; then + version_tag="latest" + need_update="true" + if [ "$ENABLE_HASH_GATE" = "true" ] && [ "$source_sha" = "$LAST_HASH_VALUE" ]; then + need_update="false" + fi + else + version_tag="$(echo "$BRANCH" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n 1)" + if [ -z "$version_tag" ]; then + echo "Branch name does not contain a valid version number (x.x.x): $BRANCH" + exit 1 + fi + need_update="true" + fi + + build_matrix_json='[{"module":"pd","image_repo":"hugegraph/pd","dockerfile":"./hugegraph-pd/Dockerfile","smoke_test":false},{"module":"store","image_repo":"hugegraph/store","dockerfile":"./hugegraph-store/Dockerfile","smoke_test":false},{"module":"server-hstore","image_repo":"hugegraph/server","dockerfile":"./hugegraph-server/Dockerfile-hstore","smoke_test":false},{"module":"server-standalone","image_repo":"hugegraph/hugegraph","dockerfile":"./hugegraph-server/Dockerfile","smoke_test":true}]' + + { + echo "source_sha=$source_sha" + echo "checkout_ref=$checkout_ref" + echo "version_tag=$version_tag" + echo "need_update=$need_update" + echo "build_matrix_json=$build_matrix_json" + } >> "$GITHUB_OUTPUT" + + integration_precheck: + needs: prepare + if: ${{ needs.prepare.outputs.need_update == 'true' && inputs.strict_mode }} + runs-on: ubuntu-latest + env: + REPOSITORY_URL: ${{ inputs.repository_url }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + MVN_ARGS: ${{ inputs.mvn_args }} + WAIT_TIMEOUT_SEC: ${{ inputs.wait_timeout_sec }} + steps: + - name: Validate wait timeout + run: | + if ! [[ "$WAIT_TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$WAIT_TIMEOUT_SEC" -lt 30 ] || [ "$WAIT_TIMEOUT_SEC" -gt 1800 ]; then + echo "Invalid wait_timeout_sec: $WAIT_TIMEOUT_SEC. Expected integer between 30 and 1800." + exit 1 + fi + + - name: Checkout source + uses: actions/checkout@v4 + with: + repository: ${{ env.REPOSITORY_URL }} + ref: ${{ env.SOURCE_SHA }} + fetch-depth: 2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Precheck cleanup + run: | + docker system prune -af || true + docker builder prune -af || true + + - name: Build x86 PD image for integration check + uses: docker/build-push-action@v7 + with: + context: . + file: ./hugegraph-pd/Dockerfile + platforms: linux/amd64 + load: true + tags: hg-ci/pd:precheck + cache-from: type=gha,scope=pd-store-server-pd + cache-to: type=gha,scope=pd-store-server-pd,mode=min + build-args: ${{ env.MVN_ARGS }} + + - name: Build x86 Store image for integration check + uses: docker/build-push-action@v7 + with: + context: . + file: ./hugegraph-store/Dockerfile + platforms: linux/amd64 + load: true + tags: hg-ci/store:precheck + cache-from: type=gha,scope=pd-store-server-store + cache-to: type=gha,scope=pd-store-server-store,mode=min + build-args: ${{ env.MVN_ARGS }} + + - name: Build x86 Server(hstore) image for integration check + uses: docker/build-push-action@v7 + with: + context: . + file: ./hugegraph-server/Dockerfile-hstore + platforms: linux/amd64 + load: true + tags: hg-ci/server:precheck + cache-from: type=gha,scope=pd-store-server-server-hstore + cache-to: type=gha,scope=pd-store-server-server-hstore,mode=min + build-args: ${{ env.MVN_ARGS }} + + - name: Start compose stack with local images + run: | + if [ ! -f "docker/docker-compose.yml" ]; then + echo "ERROR: docker/docker-compose.yml not found in $REPOSITORY_URL@$SOURCE_SHA" + echo "Please update the compose file path in this workflow." + exit 1 + fi + + cat > /tmp/docker-compose.ci.override.yml </dev/null + curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8520/v1/health" >/dev/null + curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8080/versions" >/dev/null + + - name: Dump compose logs on failure + if: ${{ failure() }} + run: | + docker compose -p hg-ci-precheck -f docker/docker-compose.yml -f /tmp/docker-compose.ci.override.yml logs --no-color --tail=200 || true + + - name: Stop compose stack + if: ${{ always() }} + run: | + docker compose \ + -p hg-ci-precheck \ + -f docker/docker-compose.yml \ + -f /tmp/docker-compose.ci.override.yml \ + down -v --remove-orphans || true + + - name: Post-check cleanup + if: ${{ always() }} + run: | + docker system prune -af || true + docker builder prune -af || true + + publish_amd64: + needs: [prepare, integration_precheck] + if: ${{ needs.prepare.outputs.need_update == 'true' && (needs.integration_precheck.result == 'success' || needs.integration_precheck.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.build_matrix_json) }} + env: + REPOSITORY_URL: ${{ inputs.repository_url }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + VERSION_TAG: ${{ needs.prepare.outputs.version_tag }} + MVN_ARGS: ${{ inputs.mvn_args }} + steps: + - name: Resolve tags (${{ matrix.module }}) + id: tags + env: + IMAGE_REPO: ${{ matrix.image_repo }} + run: | + image_amd64="${IMAGE_REPO}:${VERSION_TAG}-amd64" + cache_scope="pd-store-server-${{ matrix.module }}" + + { + echo "image_amd64=$image_amd64" + echo "cache_scope=$cache_scope" + } >> "$GITHUB_OUTPUT" + + - name: Checkout source (${{ matrix.module }}) + uses: actions/checkout@v4 + with: + repository: ${{ env.REPOSITORY_URL }} + ref: ${{ env.SOURCE_SHA }} + fetch-depth: 2 + + - name: Set up Docker Buildx (${{ matrix.module }}) + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub (${{ matrix.module }}) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push amd64 image (${{ matrix.module }}) + uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: ${{ steps.tags.outputs.image_amd64 }} + cache-from: type=gha,scope=${{ steps.tags.outputs.cache_scope }} + cache-to: type=gha,scope=${{ steps.tags.outputs.cache_scope }},mode=max + build-args: ${{ env.MVN_ARGS }} + + - name: Smoke test standalone server amd64 + if: ${{ matrix.smoke_test }} + env: + IMAGE_URL: ${{ steps.tags.outputs.image_amd64 }} + run: | + docker rm -f graph >/dev/null 2>&1 || true + docker run -d --name=graph -p 18080:8080 "$IMAGE_URL" + sleep 20 + curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1 + docker ps -a + sleep 20 + curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1 + + - name: Cleanup smoke test container + if: ${{ always() && matrix.smoke_test }} + run: | + docker rm -f graph >/dev/null 2>&1 || true + + publish_arm64: + needs: [prepare, integration_precheck, publish_amd64] + if: ${{ needs.prepare.outputs.need_update == 'true' && needs.publish_amd64.result == 'success' && (needs.integration_precheck.result == 'success' || needs.integration_precheck.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.build_matrix_json) }} + env: + REPOSITORY_URL: ${{ inputs.repository_url }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + VERSION_TAG: ${{ needs.prepare.outputs.version_tag }} + MVN_ARGS: ${{ inputs.mvn_args }} + steps: + - name: Resolve tags (${{ matrix.module }}) + id: tags + env: + IMAGE_REPO: ${{ matrix.image_repo }} + run: | + image_arm64="${IMAGE_REPO}:${VERSION_TAG}-arm64" + cache_scope="pd-store-server-${{ matrix.module }}" + + { + echo "image_arm64=$image_arm64" + echo "cache_scope=$cache_scope" + } >> "$GITHUB_OUTPUT" + + - name: Checkout source (${{ matrix.module }}) + uses: actions/checkout@v4 + with: + repository: ${{ env.REPOSITORY_URL }} + ref: ${{ env.SOURCE_SHA }} + fetch-depth: 2 + + - name: Set up QEMU (${{ matrix.module }}) + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx (${{ matrix.module }}) + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub (${{ matrix.module }}) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push arm64 image (${{ matrix.module }}) + uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/arm64 + push: true + tags: ${{ steps.tags.outputs.image_arm64 }} + cache-from: type=gha,scope=${{ steps.tags.outputs.cache_scope }} + cache-to: type=gha,scope=${{ steps.tags.outputs.cache_scope }},mode=max + build-args: ${{ env.MVN_ARGS }} + + publish_manifest: + needs: [prepare, publish_amd64, publish_arm64] + if: ${{ needs.prepare.outputs.need_update == 'true' && needs.publish_amd64.result == 'success' && needs.publish_arm64.result == 'success' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.build_matrix_json) }} + env: + VERSION_TAG: ${{ needs.prepare.outputs.version_tag }} + steps: + - name: Resolve tags (${{ matrix.module }}) + id: tags + env: + IMAGE_REPO: ${{ matrix.image_repo }} + run: | + image_final="${IMAGE_REPO}:${VERSION_TAG}" + image_amd64="${IMAGE_REPO}:${VERSION_TAG}-amd64" + image_arm64="${IMAGE_REPO}:${VERSION_TAG}-arm64" + + { + echo "image_final=$image_final" + echo "image_amd64=$image_amd64" + echo "image_arm64=$image_arm64" + } >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx (${{ matrix.module }}) + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub (${{ matrix.module }}) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Create multi-arch manifest (${{ matrix.module }}) + run: | + docker buildx imagetools create \ + --tag "${{ steps.tags.outputs.image_final }}" \ + "${{ steps.tags.outputs.image_amd64 }}" \ + "${{ steps.tags.outputs.image_arm64 }}" + + - name: Inspect multi-arch manifest (${{ matrix.module }}) + run: | + docker buildx imagetools inspect "${{ steps.tags.outputs.image_final }}" + + update_latest_hash: + needs: [prepare, publish_manifest] + if: ${{ inputs.mode == 'latest' && inputs.enable_hash_gate && needs.prepare.outputs.need_update == 'true' && needs.publish_manifest.result == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Validate hash update inputs + env: + LAST_HASH_NAME: ${{ inputs.last_hash_name }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + run: | + set -euo pipefail + if [ -z "$LAST_HASH_NAME" ]; then + echo "last_hash_name is required when enable_hash_gate=true" + exit 1 + fi + if [ -z "$PERSONAL_ACCESS_TOKEN" ]; then + echo "PERSONAL_ACCESS_TOKEN is required to update latest hash" + exit 1 + fi + + - name: Update latest source hash variable + env: + OWNER: ${{ inputs.hash_repo_owner }} + REPO: ${{ inputs.hash_repo_name }} + LAST_HASH_NAME: ${{ inputs.last_hash_name }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + run: | + set -euo pipefail + curl --fail-with-body -sS -L -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "Authorization: Bearer $PERSONAL_ACCESS_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/actions/variables/$LAST_HASH_NAME" \ + -d '{"name":"'"$LAST_HASH_NAME"'","value":"'"$SOURCE_SHA"'"}' diff --git a/.github/workflows/publish_latest_pd_store_server_image.yml b/.github/workflows/publish_latest_pd_store_server_image.yml index 4442361..683e9dd 100644 --- a/.github/workflows/publish_latest_pd_store_server_image.yml +++ b/.github/workflows/publish_latest_pd_store_server_image.yml @@ -1,6 +1,8 @@ name: "Publish pd-store-server image(latest)" on: + schedule: + - cron: '0 23 * * *' workflow_dispatch: inputs: mvn_args: @@ -22,324 +24,18 @@ concurrency: cancel-in-progress: false jobs: - resolve_source: - runs-on: ubuntu-latest - outputs: - source_sha: ${{ steps.resolve.outputs.source_sha }} - steps: - - name: Resolve source SHA - id: resolve - run: | - source_sha="$(git ls-remote https://github.com/apache/hugegraph.git refs/heads/master | awk '{print $1}')" - if [ -z "$source_sha" ]; then - echo "Failed to resolve source SHA for apache/hugegraph master" - exit 1 - fi - echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT" - - integration_precheck: - needs: resolve_source - if: ${{ inputs.strict_mode }} - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph - SOURCE_SHA: ${{ needs.resolve_source.outputs.source_sha }} - PD_IMAGE_URL: hugegraph/pd:latest - STORE_IMAGE_URL: hugegraph/store:latest - SERVER_IMAGE_URL: hugegraph/server:latest - MVN_ARGS: ${{ github.event.inputs.mvn_args || '' }} - WAIT_TIMEOUT_SEC: ${{ github.event.inputs.wait_timeout_sec || '300' }} - - steps: - - name: Validate wait timeout - run: | - if ! [[ "$WAIT_TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$WAIT_TIMEOUT_SEC" -lt 30 ] || [ "$WAIT_TIMEOUT_SEC" -gt 1800 ]; then - echo "Invalid wait_timeout_sec: $WAIT_TIMEOUT_SEC. Expected integer between 30 and 1800." - exit 1 - fi - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.SOURCE_SHA }} - fetch-depth: 2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - with: - version: latest - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Pre-build disk usage - run: | - df -h - docker system df || true - - - name: Pre-build cleanup - run: | - docker system prune -af || true - docker builder prune -af || true - - - name: Build x86 PD image for integration check - uses: docker/build-push-action@v7 - with: - context: . - file: ./hugegraph-pd/Dockerfile - load: true - tags: ${{ env.PD_IMAGE_URL }} - cache-from: type=gha,scope=latest-pd - cache-to: type=gha,scope=latest-pd,mode=min - build-args: ${{ env.MVN_ARGS }} - - - name: Build x86 Store image for integration check - uses: docker/build-push-action@v7 - with: - context: . - file: ./hugegraph-store/Dockerfile - load: true - tags: ${{ env.STORE_IMAGE_URL }} - cache-from: type=gha,scope=latest-store - cache-to: type=gha,scope=latest-store,mode=min - build-args: ${{ env.MVN_ARGS }} - - - name: Build x86 Server image for integration check - uses: docker/build-push-action@v7 - with: - context: . - file: ./hugegraph-server/Dockerfile-hstore - load: true - tags: ${{ env.SERVER_IMAGE_URL }} - cache-from: type=gha,scope=latest-server - cache-to: type=gha,scope=latest-server,mode=min - build-args: ${{ env.MVN_ARGS }} - - - name: Start compose stack with local images - run: | - if [ ! -f "docker/docker-compose.yml" ]; then - echo "ERROR: docker/docker-compose.yml not found in $REPOSITORY_URL@$SOURCE_SHA" - echo "Please update the compose file path in this workflow." - exit 1 - fi - - cat > /tmp/docker-compose.ci.override.yml </dev/null - curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8520/v1/health" >/dev/null - curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8080/versions" >/dev/null - - - name: Dump compose logs on failure - if: ${{ failure() }} - run: | - docker compose -p hg-ci-precheck -f docker/docker-compose.yml -f /tmp/docker-compose.ci.override.yml logs --no-color --tail=200 || true - - - name: Stop compose stack - if: ${{ always() }} - run: | - docker compose \ - -p hg-ci-precheck \ - -f docker/docker-compose.yml \ - -f /tmp/docker-compose.ci.override.yml \ - down -v --remove-orphans || true - - - name: Post-check disk usage - if: ${{ always() }} - run: | - docker system df || true - df -h - - - name: Post-check cleanup - if: ${{ always() }} - run: | - docker system prune -af || true - docker builder prune -af || true - - publish_matrix: - needs: [resolve_source, integration_precheck] - if: ${{ always() && needs.resolve_source.result == 'success' && (needs.integration_precheck.result == 'success' || needs.integration_precheck.result == 'skipped') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - module: pd - image_url: hugegraph/pd:latest - dockerfile: ./hugegraph-pd/Dockerfile - host_port: 8620 - probe_path: /v1/health - skip_selfcheck: false - - module: store - image_url: hugegraph/store:latest - dockerfile: ./hugegraph-store/Dockerfile - host_port: 8520 - probe_path: /v1/health - skip_selfcheck: true - - module: server - image_url: hugegraph/server:latest - dockerfile: ./hugegraph-server/Dockerfile-hstore - host_port: 8080 - probe_path: /versions - skip_selfcheck: true - - env: - REPOSITORY_URL: apache/hugegraph - SOURCE_SHA: ${{ needs.resolve_source.outputs.source_sha }} - MVN_ARGS: ${{ github.event.inputs.mvn_args || '' }} - WAIT_TIMEOUT_SEC: ${{ github.event.inputs.wait_timeout_sec || '300' }} - - steps: - - name: Validate wait timeout (${{ matrix.module }}) - run: | - if ! [[ "$WAIT_TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$WAIT_TIMEOUT_SEC" -lt 30 ] || [ "$WAIT_TIMEOUT_SEC" -gt 1800 ]; then - echo "Invalid wait_timeout_sec: $WAIT_TIMEOUT_SEC. Expected integer between 30 and 1800." - exit 1 - fi - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.SOURCE_SHA }} - fetch-depth: 2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - with: - version: latest - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Pre-build disk usage (${{ matrix.module }}) - run: | - df -h - docker system df || true - - - name: Pre-build cleanup (${{ matrix.module }}) - run: | - docker system prune -af || true - docker builder prune -af || true - - - name: Build x86 image for self-check (${{ matrix.module }}) - if: ${{ !matrix.skip_selfcheck }} - uses: docker/build-push-action@v7 - with: - context: . - file: ${{ matrix.dockerfile }} - load: true - tags: ${{ matrix.image_url }} - cache-from: type=gha,scope=latest-${{ matrix.module }} - build-args: ${{ env.MVN_ARGS }} - - - name: Self-check x86 image (${{ matrix.module }}) - if: ${{ !matrix.skip_selfcheck }} - env: - MODULE: ${{ matrix.module }} - IMAGE_URL: ${{ matrix.image_url }} - HOST_PORT: ${{ matrix.host_port }} - PROBE_PATH: ${{ matrix.probe_path }} - run: | - compose_project="hg-ci-selfcheck-${MODULE}" - compose_override="/tmp/docker-compose.self-check-${MODULE}.yml" - - if [ ! -f "docker/docker-compose.yml" ]; then - echo "ERROR: docker/docker-compose.yml not found in $REPOSITORY_URL@$SOURCE_SHA" - echo "Please update the compose file path in this workflow." - exit 1 - fi - - cat > "$compose_override" </dev/null - echo "Self-check passed: http://127.0.0.1:$HOST_PORT$PROBE_PATH" - - - name: Dump self-check compose logs on failure (${{ matrix.module }}) - if: ${{ failure() && !matrix.skip_selfcheck }} - env: - MODULE: ${{ matrix.module }} - run: | - compose_project="hg-ci-selfcheck-${MODULE}" - compose_override="/tmp/docker-compose.self-check-${MODULE}.yml" - docker compose -p "$compose_project" -f docker/docker-compose.yml -f "$compose_override" ps || true - docker compose -p "$compose_project" -f docker/docker-compose.yml -f "$compose_override" logs --no-color --tail=200 || true - - - name: Stop self-check compose stack (${{ matrix.module }}) - if: ${{ always() && !matrix.skip_selfcheck }} - env: - MODULE: ${{ matrix.module }} - run: | - compose_project="hg-ci-selfcheck-${MODULE}" - compose_override="/tmp/docker-compose.self-check-${MODULE}.yml" - docker compose \ - -p "$compose_project" \ - -f docker/docker-compose.yml \ - -f "$compose_override" \ - down -v --remove-orphans || true - rm -f "$compose_override" - - - name: Build and push multi-arch image (${{ matrix.module }}) - uses: docker/build-push-action@v7 - with: - context: . - file: ${{ matrix.dockerfile }} - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ matrix.image_url }} - cache-from: type=gha,scope=latest-${{ matrix.module }} - cache-to: type=gha,scope=latest-${{ matrix.module }},mode=max - build-args: ${{ env.MVN_ARGS }} - - - name: Post-build disk usage (${{ matrix.module }}) - if: ${{ always() }} - run: | - docker system df || true - df -h - - - name: Post-build cleanup (${{ matrix.module }}) - if: ${{ always() }} - run: | - docker system prune -af || true - docker builder prune -af || true + publish: + uses: ./.github/workflows/_publish_pd_store_server_reusable.yml + with: + mode: latest + repository_url: apache/hugegraph + branch: master + mvn_args: ${{ github.event.inputs.mvn_args || '' }} + strict_mode: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.strict_mode != 'false' }} + wait_timeout_sec: ${{ github.event.inputs.wait_timeout_sec || '300' }} + enable_hash_gate: true + last_hash_value: ${{ vars.LAST_SERVER_HASH }} + last_hash_name: LAST_SERVER_HASH + hash_repo_owner: hugegraph + hash_repo_name: actions + secrets: inherit diff --git a/.github/workflows/publish_latest_server_image.yml b/.github/workflows/publish_latest_server_image.yml deleted file mode 100644 index 1935bf5..0000000 --- a/.github/workflows/publish_latest_server_image.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "Publish server image(latest)" - -on: - schedule: - - cron: '0 23 * * *' - workflow_dispatch: - inputs: - mvn_args: - required: false - default: '' - description: 'mvn build args, like "MAVEN_ARGS=-P stage"' - -jobs: - publish: - uses: ./.github/workflows/_publish_image_reusable.yml - with: - mode: latest - component: server - repository_url: apache/hugegraph - branch: master - build_matrix_json: | - [ - { - "module": "server", - "context": ".", - "dockerfile": "./hugegraph-server/Dockerfile", - "image_repo_latest": "hugegraph/hugegraph", - "image_repo_release": "hugegraph/hugegraph", - "platforms_latest": "linux/amd64,linux/arm64", - "platforms_release": "linux/amd64,linux/arm64", - "smoke_test": true, - "smoke_test_cmd": "docker rm -f graph >/dev/null 2>&1 || true; docker run -itd --name=graph -p 18080:8080 $IMAGE_URL; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a" - } - ] - use_mvn_args: true - mvn_args: ${{ github.event.inputs.mvn_args || '' }} - enable_hash_gate: true - last_hash_value: ${{ vars.LAST_SERVER_HASH }} - last_hash_name: LAST_SERVER_HASH - hash_repo_owner: hugegraph - hash_repo_name: actions - secrets: inherit diff --git a/.github/workflows/publish_release_pd_store_server_image.yml b/.github/workflows/publish_release_pd_store_server_image.yml new file mode 100644 index 0000000..93c5bf0 --- /dev/null +++ b/.github/workflows/publish_release_pd_store_server_image.yml @@ -0,0 +1,39 @@ +name: "Publish pd-store-server image(release)" + +on: + workflow_dispatch: + inputs: + branch: + required: true + default: '' + description: 'The branch name should be like *-x.x.x, for example release-1.0.0' + mvn_args: + required: false + default: '' + description: 'mvn build args, like "MAVEN_ARGS=-P stage"' + strict_mode: + type: boolean + required: false + default: true + description: 'whether integration precheck is mandatory before publish' + wait_timeout_sec: + required: false + default: '300' + description: 'docker compose wait timeout in seconds for precheck/self-check' + +concurrency: + group: publish-release-pd-store-server + cancel-in-progress: false + +jobs: + publish: + uses: ./.github/workflows/_publish_pd_store_server_reusable.yml + with: + mode: release + repository_url: apache/hugegraph + branch: ${{ inputs.branch }} + mvn_args: ${{ inputs.mvn_args }} + strict_mode: ${{ inputs.strict_mode }} + wait_timeout_sec: ${{ inputs.wait_timeout_sec || '300' }} + enable_hash_gate: false + secrets: inherit diff --git a/.github/workflows/publish_release_server_image.yml b/.github/workflows/publish_release_server_image.yml deleted file mode 100644 index aa39dd7..0000000 --- a/.github/workflows/publish_release_server_image.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Publish server image(release)" - -on: - workflow_dispatch: - inputs: - branch: - required: true - default: '' - description: 'The branch name should be like *-x.x.x, for example release-1.0.0' - mvn_args: - required: false - default: '' - description: 'mvn build args, like "MAVEN_ARGS=-P stage"' - -jobs: - publish: - uses: ./.github/workflows/_publish_image_reusable.yml - with: - mode: release - component: server - repository_url: apache/hugegraph - branch: ${{ inputs.branch }} - build_matrix_json: | - [ - { - "module": "server", - "context": ".", - "dockerfile": "./hugegraph-server/Dockerfile", - "image_repo_latest": "hugegraph/hugegraph", - "image_repo_release": "hugegraph/hugegraph", - "platforms_latest": "linux/amd64,linux/arm64", - "platforms_release": "linux/amd64,linux/arm64", - "smoke_test": true, - "smoke_test_cmd": "docker rm -f graph >/dev/null 2>&1 || true; docker run -itd --name=graph -p 18080:8080 $IMAGE_URL; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a" - } - ] - use_mvn_args: true - mvn_args: ${{ inputs.mvn_args }} - secrets: inherit diff --git a/README.md b/README.md index 496c1e6..83b3e5e 100644 --- a/README.md +++ b/README.md @@ -22,20 +22,15 @@ The image publishing workflows are intentionally split into two layers: +------------+---------------+ | v - .github/workflows/_publish_image_reusable.yml + reusable workflow implementation | - +-----------------+-------------------+ - | | - v v - resolve mode / branch / tag build matrix per module - | | - v v - optional hash gate optional smoke test - | | - +-----------------+-------------------+ - | - v - docker build/push + +-----------------+----------------------------+ + | | + v v +_publish_image_reusable.yml _publish_pd_store_server_reusable.yml + | | + v v +standard single-image flow pd/store/server specialized flow ``` The two publishing modes behave differently: @@ -64,13 +59,14 @@ Although the `latest` and `release` wrappers look similar, they encode different - It expects a release branch and publishes that branch as a versioned image. - It should run even if the source is unchanged, because the operator is explicitly asking for a release publication. -The common build logic already lives in [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml), so the thin wrapper files mainly exist to keep the trigger semantics obvious and safe. +Most wrappers use [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml). +The pd/store/server wrappers use [`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml), which adds integration precheck plus staged amd64/arm64 publish and manifest merge. ## Reusable Workflow Responsibilities -[`_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml) is the real implementation layer. +Reusable workflows are the real implementation layer. -It handles: +`_publish_image_reusable.yml` handles the standard image flow: - resolving `latest` vs `release` mode - checking out the correct source commit @@ -81,6 +77,14 @@ It handles: - pushing the final image - updating the latest-hash variable for `latest` mode only +`_publish_pd_store_server_reusable.yml` handles the pd/store/server flow: + +- shared source SHA resolution and latest hash gate +- strict integration precheck for pd/store/server(hstore) +- staged image publication with `*-amd64` then `*-arm64` +- manifest merge to final tag (`latest` or release version) +- standalone server smoke test for `hugegraph/hugegraph` + Wrapper workflows only provide the source repository, branch, matrix definition, and mode-specific inputs. ## How To Extend @@ -89,7 +93,7 @@ When adding a new image publishing workflow, follow the same pattern: 1. Create a thin `publish_latest_*.yml` wrapper if the image needs scheduled or hash-gated automatic publishing. 2. Create a matching `publish_release_*.yml` wrapper if the image also needs manual release publishing. -3. Put all shared build behavior into `_publish_image_reusable.yml` instead of duplicating Docker or checkout logic. +3. Put shared build behavior into the appropriate reusable workflow instead of duplicating Docker or checkout logic. 4. Put image-specific values in the wrapper via `build_matrix_json`, especially: - module name - Dockerfile path @@ -109,15 +113,13 @@ Keep a dedicated workflow file when the publishing flow has materially different - different trigger semantics that do not fit the `latest` / `release` split - legacy workflows that still require bespoke setup -For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) has a more customized precheck-oriented flow than the standard image publishers. +For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) and [`.github/workflows/publish_release_pd_store_server_image.yml`](./.github/workflows/publish_release_pd_store_server_image.yml) use a dedicated reusable workflow for specialized precheck and publish sequencing. ## Current Workflow Map - Standard reusable publish path: - [`.github/workflows/publish_latest_loader_image.yml`](./.github/workflows/publish_latest_loader_image.yml) - [`.github/workflows/publish_release_loader_image.yml`](./.github/workflows/publish_release_loader_image.yml) - - [`.github/workflows/publish_latest_server_image.yml`](./.github/workflows/publish_latest_server_image.yml) - - [`.github/workflows/publish_release_server_image.yml`](./.github/workflows/publish_release_server_image.yml) - [`.github/workflows/publish_latest_hubble_image.yml`](./.github/workflows/publish_latest_hubble_image.yml) - [`.github/workflows/publish_release_hubble_image.yml`](./.github/workflows/publish_release_hubble_image.yml) - [`.github/workflows/publish_latest_vermeer_image.yml`](./.github/workflows/publish_latest_vermeer_image.yml) @@ -125,14 +127,19 @@ For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.g - [`.github/workflows/publish_latest_ai_image.yml`](./.github/workflows/publish_latest_ai_image.yml) - [`.github/workflows/publish_release_ai_image.yml`](./.github/workflows/publish_release_ai_image.yml) -- Legacy or special-case workflows: +- Dedicated reusable publish path: + - [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) + - [`.github/workflows/publish_release_pd_store_server_image.yml`](./.github/workflows/publish_release_pd_store_server_image.yml) + - [`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml) + +- Other legacy or special-case workflows: - [`.github/workflows/publish_hugegraph_hubble.yml`](./.github/workflows/publish_hugegraph_hubble.yml) - [`.github/workflows/publish_computer_image.yml`](./.github/workflows/publish_computer_image.yml) - - [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) ## Practical Notes - `latest` workflows typically run on a schedule and accept manual dispatch. - `release` workflows typically accept only manual dispatch with a branch input. -- Most image workflows inherit credentials and settings through the reusable workflow. -- If you change the shared publishing behavior, update `_publish_image_reusable.yml` first and then adjust wrappers only where their inputs change. +- Most image workflows inherit credentials and settings through a reusable workflow. +- If you change shared standard behavior, update `_publish_image_reusable.yml` first. +- If you change pd/store/server behavior, update `_publish_pd_store_server_reusable.yml` first. From c7676c7721e739fe2a4a4e124fca232df6b6e2a1 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 22 Mar 2026 19:33:10 +0800 Subject: [PATCH 2/7] fix(workflows): stabilize cache export and reuse mvn layers - switch cache export to mode=min with ignore-error to avoid hard failures - add shared plus module cache scopes for better cross-module Maven layer reuse - separate amd64 and arm64 cache scopes to reduce cross-arch cache contention - add upstream refactor notes for Dockerfile and artifact-based build dedup --- .../_publish_pd_store_server_reusable.yml | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml index bb52942..52dd3e2 100644 --- a/.github/workflows/_publish_pd_store_server_reusable.yml +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -171,8 +171,12 @@ jobs: platforms: linux/amd64 load: true tags: hg-ci/pd:precheck - cache-from: type=gha,scope=pd-store-server-pd - cache-to: type=gha,scope=pd-store-server-pd,mode=min + cache-from: | + type=gha,scope=pd-store-server-shared-amd64 + type=gha,scope=pd-store-server-pd-amd64 + cache-to: | + type=gha,scope=pd-store-server-shared-amd64,mode=min,ignore-error=true + type=gha,scope=pd-store-server-pd-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Build x86 Store image for integration check @@ -183,8 +187,12 @@ jobs: platforms: linux/amd64 load: true tags: hg-ci/store:precheck - cache-from: type=gha,scope=pd-store-server-store - cache-to: type=gha,scope=pd-store-server-store,mode=min + cache-from: | + type=gha,scope=pd-store-server-shared-amd64 + type=gha,scope=pd-store-server-store-amd64 + cache-to: | + type=gha,scope=pd-store-server-shared-amd64,mode=min,ignore-error=true + type=gha,scope=pd-store-server-store-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Build x86 Server(hstore) image for integration check @@ -195,8 +203,12 @@ jobs: platforms: linux/amd64 load: true tags: hg-ci/server:precheck - cache-from: type=gha,scope=pd-store-server-server-hstore - cache-to: type=gha,scope=pd-store-server-server-hstore,mode=min + cache-from: | + type=gha,scope=pd-store-server-shared-amd64 + type=gha,scope=pd-store-server-server-hstore-amd64 + cache-to: | + type=gha,scope=pd-store-server-shared-amd64,mode=min,ignore-error=true + type=gha,scope=pd-store-server-server-hstore-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Start compose stack with local images @@ -274,11 +286,13 @@ jobs: IMAGE_REPO: ${{ matrix.image_repo }} run: | image_amd64="${IMAGE_REPO}:${VERSION_TAG}-amd64" - cache_scope="pd-store-server-${{ matrix.module }}" + module_cache_scope="pd-store-server-${{ matrix.module }}-amd64" + shared_cache_scope="pd-store-server-shared-amd64" { echo "image_amd64=$image_amd64" - echo "cache_scope=$cache_scope" + echo "module_cache_scope=$module_cache_scope" + echo "shared_cache_scope=$shared_cache_scope" } >> "$GITHUB_OUTPUT" - name: Checkout source (${{ matrix.module }}) @@ -307,8 +321,12 @@ jobs: platforms: linux/amd64 push: true tags: ${{ steps.tags.outputs.image_amd64 }} - cache-from: type=gha,scope=${{ steps.tags.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.tags.outputs.cache_scope }},mode=max + cache-from: | + type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} + type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-to: | + type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }},mode=min,ignore-error=true + type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Smoke test standalone server amd64 @@ -349,11 +367,13 @@ jobs: IMAGE_REPO: ${{ matrix.image_repo }} run: | image_arm64="${IMAGE_REPO}:${VERSION_TAG}-arm64" - cache_scope="pd-store-server-${{ matrix.module }}" + module_cache_scope="pd-store-server-${{ matrix.module }}-arm64" + shared_cache_scope="pd-store-server-shared-arm64" { echo "image_arm64=$image_arm64" - echo "cache_scope=$cache_scope" + echo "module_cache_scope=$module_cache_scope" + echo "shared_cache_scope=$shared_cache_scope" } >> "$GITHUB_OUTPUT" - name: Checkout source (${{ matrix.module }}) @@ -385,8 +405,12 @@ jobs: platforms: linux/arm64 push: true tags: ${{ steps.tags.outputs.image_arm64 }} - cache-from: type=gha,scope=${{ steps.tags.outputs.cache_scope }} - cache-to: type=gha,scope=${{ steps.tags.outputs.cache_scope }},mode=max + cache-from: | + type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} + type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-to: | + type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }},mode=min,ignore-error=true + type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} publish_manifest: From d92d6d8f7038af7fd3bb504cb8f1b02db9f15878 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 22 Mar 2026 19:46:23 +0800 Subject: [PATCH 3/7] chore(workflows): apply review clarity fixes - remove unused checkout_ref output and related shell assignment - serialize build_matrix_json with multiline output to reduce quoting risk - clarify hstore wording in README for external readers - keep workflow map list focused on triggerable wrapper workflows --- .../_publish_pd_store_server_reusable.yml | 37 +++++++++++++++---- README.md | 3 +- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml index 52dd3e2..0173b45 100644 --- a/.github/workflows/_publish_pd_store_server_reusable.yml +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -70,7 +70,6 @@ jobs: outputs: need_update: ${{ steps.prepare.outputs.need_update }} source_sha: ${{ steps.prepare.outputs.source_sha }} - checkout_ref: ${{ steps.prepare.outputs.checkout_ref }} version_tag: ${{ steps.prepare.outputs.version_tag }} build_matrix_json: ${{ steps.prepare.outputs.build_matrix_json }} steps: @@ -96,8 +95,6 @@ jobs: exit 1 fi - checkout_ref="$source_sha" - if [ "$MODE" = "latest" ]; then version_tag="latest" need_update="true" @@ -113,14 +110,40 @@ jobs: need_update="true" fi - build_matrix_json='[{"module":"pd","image_repo":"hugegraph/pd","dockerfile":"./hugegraph-pd/Dockerfile","smoke_test":false},{"module":"store","image_repo":"hugegraph/store","dockerfile":"./hugegraph-store/Dockerfile","smoke_test":false},{"module":"server-hstore","image_repo":"hugegraph/server","dockerfile":"./hugegraph-server/Dockerfile-hstore","smoke_test":false},{"module":"server-standalone","image_repo":"hugegraph/hugegraph","dockerfile":"./hugegraph-server/Dockerfile","smoke_test":true}]' - { echo "source_sha=$source_sha" - echo "checkout_ref=$checkout_ref" echo "version_tag=$version_tag" echo "need_update=$need_update" - echo "build_matrix_json=$build_matrix_json" + echo "build_matrix_json<> "$GITHUB_OUTPUT" integration_precheck: diff --git a/README.md b/README.md index 83b3e5e..d7002cb 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Reusable workflows are the real implementation layer. `_publish_pd_store_server_reusable.yml` handles the pd/store/server flow: - shared source SHA resolution and latest hash gate -- strict integration precheck for pd/store/server(hstore) +- strict integration precheck for pd/store/server (hstore backend, `hugegraph/server`) - staged image publication with `*-amd64` then `*-arm64` - manifest merge to final tag (`latest` or release version) - standalone server smoke test for `hugegraph/hugegraph` @@ -130,7 +130,6 @@ For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.g - Dedicated reusable publish path: - [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) - [`.github/workflows/publish_release_pd_store_server_image.yml`](./.github/workflows/publish_release_pd_store_server_image.yml) - - [`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml) - Other legacy or special-case workflows: - [`.github/workflows/publish_hugegraph_hubble.yml`](./.github/workflows/publish_hugegraph_hubble.yml) From 981ef3558f036f1722bbfdbe534326825d9ad948 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 22 Mar 2026 20:27:47 +0800 Subject: [PATCH 4/7] fix(workflows): harden strict mode and smoke flow - make latest strict_mode expression robust to both boolean and string false - smoke-test standalone amd64 image locally before pushing registry tag - keep non-smoke modules on direct build-and-push path - clarify README wrapper inputs and matrix ownership wording --- .../_publish_pd_store_server_reusable.yml | 25 +++++++++++++++++++ .../publish_latest_pd_store_server_image.yml | 2 +- README.md | 3 ++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml index 0173b45..9ef022a 100644 --- a/.github/workflows/_publish_pd_store_server_reusable.yml +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -336,7 +336,25 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and load amd64 image for smoke test (${{ matrix.module }}) + if: ${{ matrix.smoke_test }} + uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + load: true + tags: ${{ steps.tags.outputs.image_amd64 }} + cache-from: | + type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} + type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-to: | + type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }},mode=min,ignore-error=true + type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true + build-args: ${{ env.MVN_ARGS }} + - name: Build and push amd64 image (${{ matrix.module }}) + if: ${{ !matrix.smoke_test }} uses: docker/build-push-action@v7 with: context: . @@ -365,6 +383,13 @@ jobs: sleep 20 curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1 + - name: Push tested amd64 image (${{ matrix.module }}) + if: ${{ matrix.smoke_test }} + env: + IMAGE_URL: ${{ steps.tags.outputs.image_amd64 }} + run: | + docker push "$IMAGE_URL" + - name: Cleanup smoke test container if: ${{ always() && matrix.smoke_test }} run: | diff --git a/.github/workflows/publish_latest_pd_store_server_image.yml b/.github/workflows/publish_latest_pd_store_server_image.yml index 683e9dd..1ac38ff 100644 --- a/.github/workflows/publish_latest_pd_store_server_image.yml +++ b/.github/workflows/publish_latest_pd_store_server_image.yml @@ -31,7 +31,7 @@ jobs: repository_url: apache/hugegraph branch: master mvn_args: ${{ github.event.inputs.mvn_args || '' }} - strict_mode: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.strict_mode != 'false' }} + strict_mode: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.strict_mode == true || github.event.inputs.strict_mode == 'true' }} wait_timeout_sec: ${{ github.event.inputs.wait_timeout_sec || '300' }} enable_hash_gate: true last_hash_value: ${{ vars.LAST_SERVER_HASH }} diff --git a/README.md b/README.md index d7002cb..732afe1 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,8 @@ Reusable workflows are the real implementation layer. - manifest merge to final tag (`latest` or release version) - standalone server smoke test for `hugegraph/hugegraph` -Wrapper workflows only provide the source repository, branch, matrix definition, and mode-specific inputs. +Wrapper workflows provide the source repository, branch, and mode-specific inputs. +Standard wrappers may also pass `build_matrix_json`, while the pd/store/server matrix is defined inside `_publish_pd_store_server_reusable.yml`. ## How To Extend From b094fd2b763afbc4244b3a56c8c181cad953f903 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 22 Mar 2026 21:20:48 +0800 Subject: [PATCH 5/7] fix(workflows): tighten cache writes and readme wording - change shared cache to read-only by removing shared scope from cache-to - keep per-module cache export with mode=min and ignore-error=true - preserve shared scope in cache-from for best-effort reuse - fix README grammar for scheduled/hash-gated wrapper guidance --- .../_publish_pd_store_server_reusable.yml | 24 +++++-------------- README.md | 5 ++-- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml index 9ef022a..46f0e18 100644 --- a/.github/workflows/_publish_pd_store_server_reusable.yml +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -197,9 +197,7 @@ jobs: cache-from: | type=gha,scope=pd-store-server-shared-amd64 type=gha,scope=pd-store-server-pd-amd64 - cache-to: | - type=gha,scope=pd-store-server-shared-amd64,mode=min,ignore-error=true - type=gha,scope=pd-store-server-pd-amd64,mode=min,ignore-error=true + cache-to: type=gha,scope=pd-store-server-pd-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Build x86 Store image for integration check @@ -213,9 +211,7 @@ jobs: cache-from: | type=gha,scope=pd-store-server-shared-amd64 type=gha,scope=pd-store-server-store-amd64 - cache-to: | - type=gha,scope=pd-store-server-shared-amd64,mode=min,ignore-error=true - type=gha,scope=pd-store-server-store-amd64,mode=min,ignore-error=true + cache-to: type=gha,scope=pd-store-server-store-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Build x86 Server(hstore) image for integration check @@ -229,9 +225,7 @@ jobs: cache-from: | type=gha,scope=pd-store-server-shared-amd64 type=gha,scope=pd-store-server-server-hstore-amd64 - cache-to: | - type=gha,scope=pd-store-server-shared-amd64,mode=min,ignore-error=true - type=gha,scope=pd-store-server-server-hstore-amd64,mode=min,ignore-error=true + cache-to: type=gha,scope=pd-store-server-server-hstore-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Start compose stack with local images @@ -348,9 +342,7 @@ jobs: cache-from: | type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} - cache-to: | - type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }},mode=min,ignore-error=true - type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true + cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Build and push amd64 image (${{ matrix.module }}) @@ -365,9 +357,7 @@ jobs: cache-from: | type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} - cache-to: | - type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }},mode=min,ignore-error=true - type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true + cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} - name: Smoke test standalone server amd64 @@ -456,9 +446,7 @@ jobs: cache-from: | type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} - cache-to: | - type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }},mode=min,ignore-error=true - type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true + cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} publish_manifest: diff --git a/README.md b/README.md index 732afe1..40ebf31 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,8 @@ Although the `latest` and `release` wrappers look similar, they encode different - It expects a release branch and publishes that branch as a versioned image. - It should run even if the source is unchanged, because the operator is explicitly asking for a release publication. -Most wrappers use [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml). +Most wrappers use [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml). + The pd/store/server wrappers use [`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml), which adds integration precheck plus staged amd64/arm64 publish and manifest merge. ## Reusable Workflow Responsibilities @@ -92,7 +93,7 @@ Standard wrappers may also pass `build_matrix_json`, while the pd/store/server m When adding a new image publishing workflow, follow the same pattern: -1. Create a thin `publish_latest_*.yml` wrapper if the image needs scheduled or hash-gated automatic publishing. +1. Create a thin `publish_latest_*.yml` wrapper if the image needs to be scheduled or hash-gated for automatic publishing. 2. Create a matching `publish_release_*.yml` wrapper if the image also needs manual release publishing. 3. Put shared build behavior into the appropriate reusable workflow instead of duplicating Docker or checkout logic. 4. Put image-specific values in the wrapper via `build_matrix_json`, especially: From 5106dfa21122d2d4211f16801a2e92eebcb3c272 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 22 Mar 2026 21:29:55 +0800 Subject: [PATCH 6/7] fix(workflows): switch pd-store-server cache to module-only - remove unused shared cache scopes from cache-from across precheck and publish jobs - keep module-scoped cache-from/cache-to only for amd64 and arm64 builds - preserve min-sized cache export with ignore-error for stability - simplify cache behavior to match actual producer/consumer flow --- .../_publish_pd_store_server_reusable.yml | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml index 46f0e18..b259bcd 100644 --- a/.github/workflows/_publish_pd_store_server_reusable.yml +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -194,9 +194,7 @@ jobs: platforms: linux/amd64 load: true tags: hg-ci/pd:precheck - cache-from: | - type=gha,scope=pd-store-server-shared-amd64 - type=gha,scope=pd-store-server-pd-amd64 + cache-from: type=gha,scope=pd-store-server-pd-amd64 cache-to: type=gha,scope=pd-store-server-pd-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} @@ -208,9 +206,7 @@ jobs: platforms: linux/amd64 load: true tags: hg-ci/store:precheck - cache-from: | - type=gha,scope=pd-store-server-shared-amd64 - type=gha,scope=pd-store-server-store-amd64 + cache-from: type=gha,scope=pd-store-server-store-amd64 cache-to: type=gha,scope=pd-store-server-store-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} @@ -222,9 +218,7 @@ jobs: platforms: linux/amd64 load: true tags: hg-ci/server:precheck - cache-from: | - type=gha,scope=pd-store-server-shared-amd64 - type=gha,scope=pd-store-server-server-hstore-amd64 + cache-from: type=gha,scope=pd-store-server-server-hstore-amd64 cache-to: type=gha,scope=pd-store-server-server-hstore-amd64,mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} @@ -304,12 +298,10 @@ jobs: run: | image_amd64="${IMAGE_REPO}:${VERSION_TAG}-amd64" module_cache_scope="pd-store-server-${{ matrix.module }}-amd64" - shared_cache_scope="pd-store-server-shared-amd64" { echo "image_amd64=$image_amd64" echo "module_cache_scope=$module_cache_scope" - echo "shared_cache_scope=$shared_cache_scope" } >> "$GITHUB_OUTPUT" - name: Checkout source (${{ matrix.module }}) @@ -339,9 +331,7 @@ jobs: platforms: linux/amd64 load: true tags: ${{ steps.tags.outputs.image_amd64 }} - cache-from: | - type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} - type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} @@ -354,9 +344,7 @@ jobs: platforms: linux/amd64 push: true tags: ${{ steps.tags.outputs.image_amd64 }} - cache-from: | - type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} - type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} @@ -406,12 +394,10 @@ jobs: run: | image_arm64="${IMAGE_REPO}:${VERSION_TAG}-arm64" module_cache_scope="pd-store-server-${{ matrix.module }}-arm64" - shared_cache_scope="pd-store-server-shared-arm64" { echo "image_arm64=$image_arm64" echo "module_cache_scope=$module_cache_scope" - echo "shared_cache_scope=$shared_cache_scope" } >> "$GITHUB_OUTPUT" - name: Checkout source (${{ matrix.module }}) @@ -443,9 +429,7 @@ jobs: platforms: linux/arm64 push: true tags: ${{ steps.tags.outputs.image_arm64 }} - cache-from: | - type=gha,scope=${{ steps.tags.outputs.shared_cache_scope }} - type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true build-args: ${{ env.MVN_ARGS }} From 006c8d0e547316033c0395318babe0c0e586ab4b Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 22 Mar 2026 23:29:14 +0800 Subject: [PATCH 7/7] fix(workflows): clean temp arch tags after merge - add Docker Hub tag cleanup after successful manifest publish in pd/store/server flow - keep partial amd64 availability when arm64 fails by cleaning only in publish_manifest job - document the critical pd/store/server release pipeline with an ASCII workflow diagram - align AGENTS.md with the staged dual-arch publish and cleanup semantics --- .../_publish_pd_store_server_reusable.yml | 58 +++++++++++++++++++ AGENTS.md | 9 ++- README.md | 43 ++++++++++++++ 3 files changed, 107 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml index b259bcd..5cb73d0 100644 --- a/.github/workflows/_publish_pd_store_server_reusable.yml +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -481,6 +481,64 @@ jobs: run: | docker buildx imagetools inspect "${{ steps.tags.outputs.image_final }}" + - name: Delete temporary arch tags after manifest publish (${{ matrix.module }}) + env: + IMAGE_REPO: ${{ matrix.image_repo }} + VERSION_TAG: ${{ env.VERSION_TAG }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + set -euo pipefail + + namespace="${IMAGE_REPO%%/*}" + repository="${IMAGE_REPO#*/}" + if [ "$namespace" = "$repository" ]; then + echo "Invalid image repo format: $IMAGE_REPO" + exit 1 + fi + + auth_token="$( + curl --fail-with-body -sS -X POST "https://hub.docker.com/v2/users/login/" \ + -H "Content-Type: application/json" \ + -d '{"username":"'"$DOCKERHUB_USERNAME"'","password":"'"$DOCKERHUB_PASSWORD"'"}' \ + | jq -r '.token' + )" + if [ -z "$auth_token" ] || [ "$auth_token" = "null" ]; then + echo "Failed to get Docker Hub auth token" + exit 1 + fi + + delete_tag_with_retry() { + local tag="$1" + local attempt=1 + local max_attempts=5 + while [ "$attempt" -le "$max_attempts" ]; do + status_code="$( + curl -sS -o /tmp/dockerhub-delete-response.txt -w "%{http_code}" -X DELETE \ + -H "Authorization: JWT $auth_token" \ + "https://hub.docker.com/v2/repositories/${namespace}/${repository}/tags/${tag}/" + )" + + if [ "$status_code" = "204" ] || [ "$status_code" = "404" ]; then + echo "Tag ${IMAGE_REPO}:${tag} delete status: $status_code" + return 0 + fi + + if [ "$attempt" -lt "$max_attempts" ]; then + echo "Delete ${IMAGE_REPO}:${tag} failed with HTTP ${status_code}, retrying (${attempt}/${max_attempts})" + sleep $((attempt * 5)) + fi + attempt=$((attempt + 1)) + done + + echo "Delete ${IMAGE_REPO}:${tag} failed after ${max_attempts} attempts" + cat /tmp/dockerhub-delete-response.txt || true + return 1 + } + + delete_tag_with_retry "${VERSION_TAG}-amd64" + delete_tag_with_retry "${VERSION_TAG}-arm64" + update_latest_hash: needs: [prepare, publish_manifest] if: ${{ inputs.mode == 'latest' && inputs.enable_hash_gate && needs.prepare.outputs.need_update == 'true' && needs.publish_manifest.result == 'success' }} diff --git a/AGENTS.md b/AGENTS.md index fb76b02..036912f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,8 +9,9 @@ Its main purpose is to publish Docker images, validate releases, and host small - `latest` publishing is the automated path: scheduled or manually triggered, with hash gating to skip unchanged sources. - `release` publishing is the manual path: it publishes from a versioned branch and should run even if the source is unchanged. -- Shared image publishing logic lives in [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml). -- Thin `publish_latest_*.yml` and `publish_release_*.yml` files are wrappers that define trigger policy and per-image inputs. +- Most image publishers share [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml). +- `pd/store/server` uses [`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml) with strict precheck and staged amd64/arm64 -> manifest flow. +- In the pd/store/server path, temporary `*-amd64` and `*-arm64` tags are cleaned only after a successful manifest publish. ## Editing Rules @@ -18,6 +19,9 @@ Its main purpose is to publish Docker images, validate releases, and host small - Keep wrapper workflows thin and explicit. - Do not merge `latest` and `release` wrappers unless the trigger semantics are truly identical. - Keep special-case workflows separate when they need extra prechecks, custom ordering, or non-standard release flow. +- For pd/store/server changes, preserve this intent: + - arm64 failure should not erase already published amd64 artifacts + - only full dual-arch success should trigger manifest + temporary tag cleanup ## Important Files @@ -31,4 +35,3 @@ Its main purpose is to publish Docker images, validate releases, and host small - Read the relevant workflow and the reusable workflow together. - Preserve existing trigger semantics unless the task explicitly asks for a behavioral change. - Check whether the workflow is a standard publisher or a legacy / special-case flow before refactoring. - diff --git a/README.md b/README.md index 40ebf31..ffc511a 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,48 @@ The two publishing modes behave differently: - always publishes when invoked - derives the image tag from the release branch version +## Critical Path: PD/Store/Server + +`pd/store/server` is the most important publishing flow in this repository and uses a dedicated reusable workflow: +[`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml). + +```text + source branch (master / release-x.y.z) + | + v + prepare job + (resolve source SHA, version tag, hash gate) + | + v + integration_precheck (optional) + (compose health check for pd/store/server-hstore) + | + v + publish_amd64 (matrix x4 modules) + +-------------------------------------------------+ + | pd | store | server-hstore | server-standalone | + +-------------------------------------------------+ + push x.y.z-amd64 (or latest-amd64) + | + v + publish_arm64 (matrix x4 modules) + push x.y.z-arm64 (or latest-arm64) + | + v + publish_manifest (matrix x4 modules) + merge amd64+arm64 => x.y.z (or latest) manifest + then delete temporary -amd64 / -arm64 tags + | + v + update_latest_hash (latest mode only, optional) +``` + +Tag behavior: + +- If only amd64 is published and arm64 fails, manifest is not created and `*-amd64` remains available. +- If both amd64 and arm64 succeed, manifest publish runs and then removes temporary `*-amd64` and `*-arm64` tags. +- End users should primarily use `latest` or release version tags (`x.y.z`). + ## Why The Wrappers Stay Split Although the `latest` and `release` wrappers look similar, they encode different release semantics. @@ -84,6 +126,7 @@ Reusable workflows are the real implementation layer. - strict integration precheck for pd/store/server (hstore backend, `hugegraph/server`) - staged image publication with `*-amd64` then `*-arm64` - manifest merge to final tag (`latest` or release version) +- remove temporary `*-amd64` and `*-arm64` tags after successful manifest publish - standalone server smoke test for `hugegraph/hugegraph` Wrapper workflows provide the source repository, branch, and mode-specific inputs.