diff --git a/.github/workflows/_publish_pd_store_server_reusable.yml b/.github/workflows/_publish_pd_store_server_reusable.yml new file mode 100644 index 0000000..b259bcd --- /dev/null +++ b/.github/workflows/_publish_pd_store_server_reusable.yml @@ -0,0 +1,518 @@ +name: "Publish pd-store-server image (reusable)" + +on: + workflow_call: + inputs: + mode: + description: "publish mode: latest or release" + required: true + type: string + repository_url: + description: "source repository in owner/name format" + required: false + default: apache/hugegraph + type: string + branch: + description: "source branch name" + required: true + type: string + mvn_args: + description: "mvn build args" + required: false + default: '' + type: string + strict_mode: + description: "whether integration precheck is mandatory before publish" + required: false + default: true + type: boolean + wait_timeout_sec: + description: "docker compose wait timeout in seconds" + required: false + default: '300' + type: string + enable_hash_gate: + description: "whether to skip latest publish if source hash unchanged" + required: false + default: false + type: boolean + last_hash_value: + description: "last published source hash" + required: false + default: '' + type: string + last_hash_name: + description: "repo variable name for latest hash" + required: false + default: '' + type: string + hash_repo_owner: + description: "owner of repo storing LAST_* hash variable" + required: false + default: hugegraph + type: string + hash_repo_name: + description: "repo name storing LAST_* hash variable" + required: false + default: actions + type: string + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_PASSWORD: + required: true + PERSONAL_ACCESS_TOKEN: + required: false + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + need_update: ${{ steps.prepare.outputs.need_update }} + source_sha: ${{ steps.prepare.outputs.source_sha }} + version_tag: ${{ steps.prepare.outputs.version_tag }} + build_matrix_json: ${{ steps.prepare.outputs.build_matrix_json }} + steps: + - name: Resolve mode and source ref + id: prepare + env: + MODE: ${{ inputs.mode }} + REPOSITORY_URL: ${{ inputs.repository_url }} + BRANCH: ${{ inputs.branch }} + ENABLE_HASH_GATE: ${{ inputs.enable_hash_gate }} + LAST_HASH_VALUE: ${{ inputs.last_hash_value }} + run: | + set -euo pipefail + + if [ "$MODE" != "latest" ] && [ "$MODE" != "release" ]; then + echo "Invalid mode: $MODE. Expected latest or release." + exit 1 + fi + + source_sha="$(git ls-remote "https://github.com/${REPOSITORY_URL}.git" "refs/heads/${BRANCH}" | awk '{print $1}')" + if [ -z "$source_sha" ]; then + echo "Failed to resolve source SHA for ${REPOSITORY_URL}@${BRANCH}" + exit 1 + fi + + if [ "$MODE" = "latest" ]; then + version_tag="latest" + need_update="true" + if [ "$ENABLE_HASH_GATE" = "true" ] && [ "$source_sha" = "$LAST_HASH_VALUE" ]; then + need_update="false" + fi + else + version_tag="$(echo "$BRANCH" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n 1)" + if [ -z "$version_tag" ]; then + echo "Branch name does not contain a valid version number (x.x.x): $BRANCH" + exit 1 + fi + need_update="true" + fi + + { + echo "source_sha=$source_sha" + echo "version_tag=$version_tag" + echo "need_update=$need_update" + echo "build_matrix_json<> "$GITHUB_OUTPUT" + + integration_precheck: + needs: prepare + if: ${{ needs.prepare.outputs.need_update == 'true' && inputs.strict_mode }} + runs-on: ubuntu-latest + env: + REPOSITORY_URL: ${{ inputs.repository_url }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + MVN_ARGS: ${{ inputs.mvn_args }} + WAIT_TIMEOUT_SEC: ${{ inputs.wait_timeout_sec }} + steps: + - name: Validate wait timeout + run: | + if ! [[ "$WAIT_TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$WAIT_TIMEOUT_SEC" -lt 30 ] || [ "$WAIT_TIMEOUT_SEC" -gt 1800 ]; then + echo "Invalid wait_timeout_sec: $WAIT_TIMEOUT_SEC. Expected integer between 30 and 1800." + exit 1 + fi + + - name: Checkout source + uses: actions/checkout@v4 + with: + repository: ${{ env.REPOSITORY_URL }} + ref: ${{ env.SOURCE_SHA }} + fetch-depth: 2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Precheck cleanup + run: | + docker system prune -af || true + docker builder prune -af || true + + - name: Build x86 PD image for integration check + uses: docker/build-push-action@v7 + with: + context: . + file: ./hugegraph-pd/Dockerfile + platforms: linux/amd64 + load: true + tags: hg-ci/pd:precheck + cache-from: type=gha,scope=pd-store-server-pd-amd64 + cache-to: type=gha,scope=pd-store-server-pd-amd64,mode=min,ignore-error=true + build-args: ${{ env.MVN_ARGS }} + + - name: Build x86 Store image for integration check + uses: docker/build-push-action@v7 + with: + context: . + file: ./hugegraph-store/Dockerfile + platforms: linux/amd64 + load: true + tags: hg-ci/store:precheck + cache-from: type=gha,scope=pd-store-server-store-amd64 + cache-to: type=gha,scope=pd-store-server-store-amd64,mode=min,ignore-error=true + build-args: ${{ env.MVN_ARGS }} + + - name: Build x86 Server(hstore) image for integration check + uses: docker/build-push-action@v7 + with: + context: . + file: ./hugegraph-server/Dockerfile-hstore + platforms: linux/amd64 + load: true + tags: hg-ci/server:precheck + cache-from: type=gha,scope=pd-store-server-server-hstore-amd64 + cache-to: type=gha,scope=pd-store-server-server-hstore-amd64,mode=min,ignore-error=true + build-args: ${{ env.MVN_ARGS }} + + - name: Start compose stack with local images + run: | + if [ ! -f "docker/docker-compose.yml" ]; then + echo "ERROR: docker/docker-compose.yml not found in $REPOSITORY_URL@$SOURCE_SHA" + echo "Please update the compose file path in this workflow." + exit 1 + fi + + cat > /tmp/docker-compose.ci.override.yml </dev/null + curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8520/v1/health" >/dev/null + curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8080/versions" >/dev/null + + - name: Dump compose logs on failure + if: ${{ failure() }} + run: | + docker compose -p hg-ci-precheck -f docker/docker-compose.yml -f /tmp/docker-compose.ci.override.yml logs --no-color --tail=200 || true + + - name: Stop compose stack + if: ${{ always() }} + run: | + docker compose \ + -p hg-ci-precheck \ + -f docker/docker-compose.yml \ + -f /tmp/docker-compose.ci.override.yml \ + down -v --remove-orphans || true + + - name: Post-check cleanup + if: ${{ always() }} + run: | + docker system prune -af || true + docker builder prune -af || true + + publish_amd64: + needs: [prepare, integration_precheck] + if: ${{ needs.prepare.outputs.need_update == 'true' && (needs.integration_precheck.result == 'success' || needs.integration_precheck.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.build_matrix_json) }} + env: + REPOSITORY_URL: ${{ inputs.repository_url }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + VERSION_TAG: ${{ needs.prepare.outputs.version_tag }} + MVN_ARGS: ${{ inputs.mvn_args }} + steps: + - name: Resolve tags (${{ matrix.module }}) + id: tags + env: + IMAGE_REPO: ${{ matrix.image_repo }} + run: | + image_amd64="${IMAGE_REPO}:${VERSION_TAG}-amd64" + module_cache_scope="pd-store-server-${{ matrix.module }}-amd64" + + { + echo "image_amd64=$image_amd64" + echo "module_cache_scope=$module_cache_scope" + } >> "$GITHUB_OUTPUT" + + - name: Checkout source (${{ matrix.module }}) + uses: actions/checkout@v4 + with: + repository: ${{ env.REPOSITORY_URL }} + ref: ${{ env.SOURCE_SHA }} + fetch-depth: 2 + + - name: Set up Docker Buildx (${{ matrix.module }}) + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub (${{ matrix.module }}) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and load amd64 image for smoke test (${{ matrix.module }}) + if: ${{ matrix.smoke_test }} + uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + load: true + tags: ${{ steps.tags.outputs.image_amd64 }} + cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true + build-args: ${{ env.MVN_ARGS }} + + - name: Build and push amd64 image (${{ matrix.module }}) + if: ${{ !matrix.smoke_test }} + uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: ${{ steps.tags.outputs.image_amd64 }} + cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true + build-args: ${{ env.MVN_ARGS }} + + - name: Smoke test standalone server amd64 + if: ${{ matrix.smoke_test }} + env: + IMAGE_URL: ${{ steps.tags.outputs.image_amd64 }} + run: | + docker rm -f graph >/dev/null 2>&1 || true + docker run -d --name=graph -p 18080:8080 "$IMAGE_URL" + sleep 20 + curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1 + docker ps -a + sleep 20 + curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1 + + - name: Push tested amd64 image (${{ matrix.module }}) + if: ${{ matrix.smoke_test }} + env: + IMAGE_URL: ${{ steps.tags.outputs.image_amd64 }} + run: | + docker push "$IMAGE_URL" + + - name: Cleanup smoke test container + if: ${{ always() && matrix.smoke_test }} + run: | + docker rm -f graph >/dev/null 2>&1 || true + + publish_arm64: + needs: [prepare, integration_precheck, publish_amd64] + if: ${{ needs.prepare.outputs.need_update == 'true' && needs.publish_amd64.result == 'success' && (needs.integration_precheck.result == 'success' || needs.integration_precheck.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.build_matrix_json) }} + env: + REPOSITORY_URL: ${{ inputs.repository_url }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + VERSION_TAG: ${{ needs.prepare.outputs.version_tag }} + MVN_ARGS: ${{ inputs.mvn_args }} + steps: + - name: Resolve tags (${{ matrix.module }}) + id: tags + env: + IMAGE_REPO: ${{ matrix.image_repo }} + run: | + image_arm64="${IMAGE_REPO}:${VERSION_TAG}-arm64" + module_cache_scope="pd-store-server-${{ matrix.module }}-arm64" + + { + echo "image_arm64=$image_arm64" + echo "module_cache_scope=$module_cache_scope" + } >> "$GITHUB_OUTPUT" + + - name: Checkout source (${{ matrix.module }}) + uses: actions/checkout@v4 + with: + repository: ${{ env.REPOSITORY_URL }} + ref: ${{ env.SOURCE_SHA }} + fetch-depth: 2 + + - name: Set up QEMU (${{ matrix.module }}) + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx (${{ matrix.module }}) + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub (${{ matrix.module }}) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push arm64 image (${{ matrix.module }}) + uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/arm64 + push: true + tags: ${{ steps.tags.outputs.image_arm64 }} + cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }} + cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true + build-args: ${{ env.MVN_ARGS }} + + publish_manifest: + needs: [prepare, publish_amd64, publish_arm64] + if: ${{ needs.prepare.outputs.need_update == 'true' && needs.publish_amd64.result == 'success' && needs.publish_arm64.result == 'success' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.build_matrix_json) }} + env: + VERSION_TAG: ${{ needs.prepare.outputs.version_tag }} + steps: + - name: Resolve tags (${{ matrix.module }}) + id: tags + env: + IMAGE_REPO: ${{ matrix.image_repo }} + run: | + image_final="${IMAGE_REPO}:${VERSION_TAG}" + image_amd64="${IMAGE_REPO}:${VERSION_TAG}-amd64" + image_arm64="${IMAGE_REPO}:${VERSION_TAG}-arm64" + + { + echo "image_final=$image_final" + echo "image_amd64=$image_amd64" + echo "image_arm64=$image_arm64" + } >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx (${{ matrix.module }}) + uses: docker/setup-buildx-action@v4 + with: + version: latest + + - name: Login to Docker Hub (${{ matrix.module }}) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Create multi-arch manifest (${{ matrix.module }}) + run: | + docker buildx imagetools create \ + --tag "${{ steps.tags.outputs.image_final }}" \ + "${{ steps.tags.outputs.image_amd64 }}" \ + "${{ steps.tags.outputs.image_arm64 }}" + + - name: Inspect multi-arch manifest (${{ matrix.module }}) + run: | + docker buildx imagetools inspect "${{ steps.tags.outputs.image_final }}" + + update_latest_hash: + needs: [prepare, publish_manifest] + if: ${{ inputs.mode == 'latest' && inputs.enable_hash_gate && needs.prepare.outputs.need_update == 'true' && needs.publish_manifest.result == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Validate hash update inputs + env: + LAST_HASH_NAME: ${{ inputs.last_hash_name }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + run: | + set -euo pipefail + if [ -z "$LAST_HASH_NAME" ]; then + echo "last_hash_name is required when enable_hash_gate=true" + exit 1 + fi + if [ -z "$PERSONAL_ACCESS_TOKEN" ]; then + echo "PERSONAL_ACCESS_TOKEN is required to update latest hash" + exit 1 + fi + + - name: Update latest source hash variable + env: + OWNER: ${{ inputs.hash_repo_owner }} + REPO: ${{ inputs.hash_repo_name }} + LAST_HASH_NAME: ${{ inputs.last_hash_name }} + SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + run: | + set -euo pipefail + curl --fail-with-body -sS -L -X PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "Authorization: Bearer $PERSONAL_ACCESS_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/actions/variables/$LAST_HASH_NAME" \ + -d '{"name":"'"$LAST_HASH_NAME"'","value":"'"$SOURCE_SHA"'"}' diff --git a/.github/workflows/publish_latest_pd_store_server_image.yml b/.github/workflows/publish_latest_pd_store_server_image.yml index 4442361..1ac38ff 100644 --- a/.github/workflows/publish_latest_pd_store_server_image.yml +++ b/.github/workflows/publish_latest_pd_store_server_image.yml @@ -1,6 +1,8 @@ name: "Publish pd-store-server image(latest)" on: + schedule: + - cron: '0 23 * * *' workflow_dispatch: inputs: mvn_args: @@ -22,324 +24,18 @@ concurrency: cancel-in-progress: false jobs: - resolve_source: - runs-on: ubuntu-latest - outputs: - source_sha: ${{ steps.resolve.outputs.source_sha }} - steps: - - name: Resolve source SHA - id: resolve - run: | - source_sha="$(git ls-remote https://github.com/apache/hugegraph.git refs/heads/master | awk '{print $1}')" - if [ -z "$source_sha" ]; then - echo "Failed to resolve source SHA for apache/hugegraph master" - exit 1 - fi - echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT" - - integration_precheck: - needs: resolve_source - if: ${{ inputs.strict_mode }} - runs-on: ubuntu-latest - env: - REPOSITORY_URL: apache/hugegraph - SOURCE_SHA: ${{ needs.resolve_source.outputs.source_sha }} - PD_IMAGE_URL: hugegraph/pd:latest - STORE_IMAGE_URL: hugegraph/store:latest - SERVER_IMAGE_URL: hugegraph/server:latest - MVN_ARGS: ${{ github.event.inputs.mvn_args || '' }} - WAIT_TIMEOUT_SEC: ${{ github.event.inputs.wait_timeout_sec || '300' }} - - steps: - - name: Validate wait timeout - run: | - if ! [[ "$WAIT_TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$WAIT_TIMEOUT_SEC" -lt 30 ] || [ "$WAIT_TIMEOUT_SEC" -gt 1800 ]; then - echo "Invalid wait_timeout_sec: $WAIT_TIMEOUT_SEC. Expected integer between 30 and 1800." - exit 1 - fi - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.SOURCE_SHA }} - fetch-depth: 2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - with: - version: latest - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Pre-build disk usage - run: | - df -h - docker system df || true - - - name: Pre-build cleanup - run: | - docker system prune -af || true - docker builder prune -af || true - - - name: Build x86 PD image for integration check - uses: docker/build-push-action@v7 - with: - context: . - file: ./hugegraph-pd/Dockerfile - load: true - tags: ${{ env.PD_IMAGE_URL }} - cache-from: type=gha,scope=latest-pd - cache-to: type=gha,scope=latest-pd,mode=min - build-args: ${{ env.MVN_ARGS }} - - - name: Build x86 Store image for integration check - uses: docker/build-push-action@v7 - with: - context: . - file: ./hugegraph-store/Dockerfile - load: true - tags: ${{ env.STORE_IMAGE_URL }} - cache-from: type=gha,scope=latest-store - cache-to: type=gha,scope=latest-store,mode=min - build-args: ${{ env.MVN_ARGS }} - - - name: Build x86 Server image for integration check - uses: docker/build-push-action@v7 - with: - context: . - file: ./hugegraph-server/Dockerfile-hstore - load: true - tags: ${{ env.SERVER_IMAGE_URL }} - cache-from: type=gha,scope=latest-server - cache-to: type=gha,scope=latest-server,mode=min - build-args: ${{ env.MVN_ARGS }} - - - name: Start compose stack with local images - run: | - if [ ! -f "docker/docker-compose.yml" ]; then - echo "ERROR: docker/docker-compose.yml not found in $REPOSITORY_URL@$SOURCE_SHA" - echo "Please update the compose file path in this workflow." - exit 1 - fi - - cat > /tmp/docker-compose.ci.override.yml </dev/null - curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8520/v1/health" >/dev/null - curl -fsS --connect-timeout 3 --max-time 8 "http://127.0.0.1:8080/versions" >/dev/null - - - name: Dump compose logs on failure - if: ${{ failure() }} - run: | - docker compose -p hg-ci-precheck -f docker/docker-compose.yml -f /tmp/docker-compose.ci.override.yml logs --no-color --tail=200 || true - - - name: Stop compose stack - if: ${{ always() }} - run: | - docker compose \ - -p hg-ci-precheck \ - -f docker/docker-compose.yml \ - -f /tmp/docker-compose.ci.override.yml \ - down -v --remove-orphans || true - - - name: Post-check disk usage - if: ${{ always() }} - run: | - docker system df || true - df -h - - - name: Post-check cleanup - if: ${{ always() }} - run: | - docker system prune -af || true - docker builder prune -af || true - - publish_matrix: - needs: [resolve_source, integration_precheck] - if: ${{ always() && needs.resolve_source.result == 'success' && (needs.integration_precheck.result == 'success' || needs.integration_precheck.result == 'skipped') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - module: pd - image_url: hugegraph/pd:latest - dockerfile: ./hugegraph-pd/Dockerfile - host_port: 8620 - probe_path: /v1/health - skip_selfcheck: false - - module: store - image_url: hugegraph/store:latest - dockerfile: ./hugegraph-store/Dockerfile - host_port: 8520 - probe_path: /v1/health - skip_selfcheck: true - - module: server - image_url: hugegraph/server:latest - dockerfile: ./hugegraph-server/Dockerfile-hstore - host_port: 8080 - probe_path: /versions - skip_selfcheck: true - - env: - REPOSITORY_URL: apache/hugegraph - SOURCE_SHA: ${{ needs.resolve_source.outputs.source_sha }} - MVN_ARGS: ${{ github.event.inputs.mvn_args || '' }} - WAIT_TIMEOUT_SEC: ${{ github.event.inputs.wait_timeout_sec || '300' }} - - steps: - - name: Validate wait timeout (${{ matrix.module }}) - run: | - if ! [[ "$WAIT_TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$WAIT_TIMEOUT_SEC" -lt 30 ] || [ "$WAIT_TIMEOUT_SEC" -gt 1800 ]; then - echo "Invalid wait_timeout_sec: $WAIT_TIMEOUT_SEC. Expected integer between 30 and 1800." - exit 1 - fi - - - name: Checkout latest - uses: actions/checkout@v4 - with: - repository: ${{ env.REPOSITORY_URL }} - ref: ${{ env.SOURCE_SHA }} - fetch-depth: 2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - with: - version: latest - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Pre-build disk usage (${{ matrix.module }}) - run: | - df -h - docker system df || true - - - name: Pre-build cleanup (${{ matrix.module }}) - run: | - docker system prune -af || true - docker builder prune -af || true - - - name: Build x86 image for self-check (${{ matrix.module }}) - if: ${{ !matrix.skip_selfcheck }} - uses: docker/build-push-action@v7 - with: - context: . - file: ${{ matrix.dockerfile }} - load: true - tags: ${{ matrix.image_url }} - cache-from: type=gha,scope=latest-${{ matrix.module }} - build-args: ${{ env.MVN_ARGS }} - - - name: Self-check x86 image (${{ matrix.module }}) - if: ${{ !matrix.skip_selfcheck }} - env: - MODULE: ${{ matrix.module }} - IMAGE_URL: ${{ matrix.image_url }} - HOST_PORT: ${{ matrix.host_port }} - PROBE_PATH: ${{ matrix.probe_path }} - run: | - compose_project="hg-ci-selfcheck-${MODULE}" - compose_override="/tmp/docker-compose.self-check-${MODULE}.yml" - - if [ ! -f "docker/docker-compose.yml" ]; then - echo "ERROR: docker/docker-compose.yml not found in $REPOSITORY_URL@$SOURCE_SHA" - echo "Please update the compose file path in this workflow." - exit 1 - fi - - cat > "$compose_override" </dev/null - echo "Self-check passed: http://127.0.0.1:$HOST_PORT$PROBE_PATH" - - - name: Dump self-check compose logs on failure (${{ matrix.module }}) - if: ${{ failure() && !matrix.skip_selfcheck }} - env: - MODULE: ${{ matrix.module }} - run: | - compose_project="hg-ci-selfcheck-${MODULE}" - compose_override="/tmp/docker-compose.self-check-${MODULE}.yml" - docker compose -p "$compose_project" -f docker/docker-compose.yml -f "$compose_override" ps || true - docker compose -p "$compose_project" -f docker/docker-compose.yml -f "$compose_override" logs --no-color --tail=200 || true - - - name: Stop self-check compose stack (${{ matrix.module }}) - if: ${{ always() && !matrix.skip_selfcheck }} - env: - MODULE: ${{ matrix.module }} - run: | - compose_project="hg-ci-selfcheck-${MODULE}" - compose_override="/tmp/docker-compose.self-check-${MODULE}.yml" - docker compose \ - -p "$compose_project" \ - -f docker/docker-compose.yml \ - -f "$compose_override" \ - down -v --remove-orphans || true - rm -f "$compose_override" - - - name: Build and push multi-arch image (${{ matrix.module }}) - uses: docker/build-push-action@v7 - with: - context: . - file: ${{ matrix.dockerfile }} - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ matrix.image_url }} - cache-from: type=gha,scope=latest-${{ matrix.module }} - cache-to: type=gha,scope=latest-${{ matrix.module }},mode=max - build-args: ${{ env.MVN_ARGS }} - - - name: Post-build disk usage (${{ matrix.module }}) - if: ${{ always() }} - run: | - docker system df || true - df -h - - - name: Post-build cleanup (${{ matrix.module }}) - if: ${{ always() }} - run: | - docker system prune -af || true - docker builder prune -af || true + publish: + uses: ./.github/workflows/_publish_pd_store_server_reusable.yml + with: + mode: latest + repository_url: apache/hugegraph + branch: master + mvn_args: ${{ github.event.inputs.mvn_args || '' }} + strict_mode: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.strict_mode == true || github.event.inputs.strict_mode == 'true' }} + wait_timeout_sec: ${{ github.event.inputs.wait_timeout_sec || '300' }} + enable_hash_gate: true + last_hash_value: ${{ vars.LAST_SERVER_HASH }} + last_hash_name: LAST_SERVER_HASH + hash_repo_owner: hugegraph + hash_repo_name: actions + secrets: inherit diff --git a/.github/workflows/publish_latest_server_image.yml b/.github/workflows/publish_latest_server_image.yml deleted file mode 100644 index 1935bf5..0000000 --- a/.github/workflows/publish_latest_server_image.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "Publish server image(latest)" - -on: - schedule: - - cron: '0 23 * * *' - workflow_dispatch: - inputs: - mvn_args: - required: false - default: '' - description: 'mvn build args, like "MAVEN_ARGS=-P stage"' - -jobs: - publish: - uses: ./.github/workflows/_publish_image_reusable.yml - with: - mode: latest - component: server - repository_url: apache/hugegraph - branch: master - build_matrix_json: | - [ - { - "module": "server", - "context": ".", - "dockerfile": "./hugegraph-server/Dockerfile", - "image_repo_latest": "hugegraph/hugegraph", - "image_repo_release": "hugegraph/hugegraph", - "platforms_latest": "linux/amd64,linux/arm64", - "platforms_release": "linux/amd64,linux/arm64", - "smoke_test": true, - "smoke_test_cmd": "docker rm -f graph >/dev/null 2>&1 || true; docker run -itd --name=graph -p 18080:8080 $IMAGE_URL; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a" - } - ] - use_mvn_args: true - mvn_args: ${{ github.event.inputs.mvn_args || '' }} - enable_hash_gate: true - last_hash_value: ${{ vars.LAST_SERVER_HASH }} - last_hash_name: LAST_SERVER_HASH - hash_repo_owner: hugegraph - hash_repo_name: actions - secrets: inherit diff --git a/.github/workflows/publish_release_pd_store_server_image.yml b/.github/workflows/publish_release_pd_store_server_image.yml new file mode 100644 index 0000000..93c5bf0 --- /dev/null +++ b/.github/workflows/publish_release_pd_store_server_image.yml @@ -0,0 +1,39 @@ +name: "Publish pd-store-server image(release)" + +on: + workflow_dispatch: + inputs: + branch: + required: true + default: '' + description: 'The branch name should be like *-x.x.x, for example release-1.0.0' + mvn_args: + required: false + default: '' + description: 'mvn build args, like "MAVEN_ARGS=-P stage"' + strict_mode: + type: boolean + required: false + default: true + description: 'whether integration precheck is mandatory before publish' + wait_timeout_sec: + required: false + default: '300' + description: 'docker compose wait timeout in seconds for precheck/self-check' + +concurrency: + group: publish-release-pd-store-server + cancel-in-progress: false + +jobs: + publish: + uses: ./.github/workflows/_publish_pd_store_server_reusable.yml + with: + mode: release + repository_url: apache/hugegraph + branch: ${{ inputs.branch }} + mvn_args: ${{ inputs.mvn_args }} + strict_mode: ${{ inputs.strict_mode }} + wait_timeout_sec: ${{ inputs.wait_timeout_sec || '300' }} + enable_hash_gate: false + secrets: inherit diff --git a/.github/workflows/publish_release_server_image.yml b/.github/workflows/publish_release_server_image.yml deleted file mode 100644 index aa39dd7..0000000 --- a/.github/workflows/publish_release_server_image.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Publish server image(release)" - -on: - workflow_dispatch: - inputs: - branch: - required: true - default: '' - description: 'The branch name should be like *-x.x.x, for example release-1.0.0' - mvn_args: - required: false - default: '' - description: 'mvn build args, like "MAVEN_ARGS=-P stage"' - -jobs: - publish: - uses: ./.github/workflows/_publish_image_reusable.yml - with: - mode: release - component: server - repository_url: apache/hugegraph - branch: ${{ inputs.branch }} - build_matrix_json: | - [ - { - "module": "server", - "context": ".", - "dockerfile": "./hugegraph-server/Dockerfile", - "image_repo_latest": "hugegraph/hugegraph", - "image_repo_release": "hugegraph/hugegraph", - "platforms_latest": "linux/amd64,linux/arm64", - "platforms_release": "linux/amd64,linux/arm64", - "smoke_test": true, - "smoke_test_cmd": "docker rm -f graph >/dev/null 2>&1 || true; docker run -itd --name=graph -p 18080:8080 $IMAGE_URL; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a; sleep 20s; curl -fsS http://127.0.0.1:18080 >/dev/null || exit 1; docker ps -a" - } - ] - use_mvn_args: true - mvn_args: ${{ inputs.mvn_args }} - secrets: inherit diff --git a/README.md b/README.md index 496c1e6..40ebf31 100644 --- a/README.md +++ b/README.md @@ -22,20 +22,15 @@ The image publishing workflows are intentionally split into two layers: +------------+---------------+ | v - .github/workflows/_publish_image_reusable.yml + reusable workflow implementation | - +-----------------+-------------------+ - | | - v v - resolve mode / branch / tag build matrix per module - | | - v v - optional hash gate optional smoke test - | | - +-----------------+-------------------+ - | - v - docker build/push + +-----------------+----------------------------+ + | | + v v +_publish_image_reusable.yml _publish_pd_store_server_reusable.yml + | | + v v +standard single-image flow pd/store/server specialized flow ``` The two publishing modes behave differently: @@ -64,13 +59,15 @@ Although the `latest` and `release` wrappers look similar, they encode different - It expects a release branch and publishes that branch as a versioned image. - It should run even if the source is unchanged, because the operator is explicitly asking for a release publication. -The common build logic already lives in [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml), so the thin wrapper files mainly exist to keep the trigger semantics obvious and safe. +Most wrappers use [`.github/workflows/_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml). + +The pd/store/server wrappers use [`.github/workflows/_publish_pd_store_server_reusable.yml`](./.github/workflows/_publish_pd_store_server_reusable.yml), which adds integration precheck plus staged amd64/arm64 publish and manifest merge. ## Reusable Workflow Responsibilities -[`_publish_image_reusable.yml`](./.github/workflows/_publish_image_reusable.yml) is the real implementation layer. +Reusable workflows are the real implementation layer. -It handles: +`_publish_image_reusable.yml` handles the standard image flow: - resolving `latest` vs `release` mode - checking out the correct source commit @@ -81,15 +78,24 @@ It handles: - pushing the final image - updating the latest-hash variable for `latest` mode only -Wrapper workflows only provide the source repository, branch, matrix definition, and mode-specific inputs. +`_publish_pd_store_server_reusable.yml` handles the pd/store/server flow: + +- shared source SHA resolution and latest hash gate +- strict integration precheck for pd/store/server (hstore backend, `hugegraph/server`) +- staged image publication with `*-amd64` then `*-arm64` +- manifest merge to final tag (`latest` or release version) +- standalone server smoke test for `hugegraph/hugegraph` + +Wrapper workflows provide the source repository, branch, and mode-specific inputs. +Standard wrappers may also pass `build_matrix_json`, while the pd/store/server matrix is defined inside `_publish_pd_store_server_reusable.yml`. ## How To Extend When adding a new image publishing workflow, follow the same pattern: -1. Create a thin `publish_latest_*.yml` wrapper if the image needs scheduled or hash-gated automatic publishing. +1. Create a thin `publish_latest_*.yml` wrapper if the image needs to be scheduled or hash-gated for automatic publishing. 2. Create a matching `publish_release_*.yml` wrapper if the image also needs manual release publishing. -3. Put all shared build behavior into `_publish_image_reusable.yml` instead of duplicating Docker or checkout logic. +3. Put shared build behavior into the appropriate reusable workflow instead of duplicating Docker or checkout logic. 4. Put image-specific values in the wrapper via `build_matrix_json`, especially: - module name - Dockerfile path @@ -109,15 +115,13 @@ Keep a dedicated workflow file when the publishing flow has materially different - different trigger semantics that do not fit the `latest` / `release` split - legacy workflows that still require bespoke setup -For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) has a more customized precheck-oriented flow than the standard image publishers. +For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) and [`.github/workflows/publish_release_pd_store_server_image.yml`](./.github/workflows/publish_release_pd_store_server_image.yml) use a dedicated reusable workflow for specialized precheck and publish sequencing. ## Current Workflow Map - Standard reusable publish path: - [`.github/workflows/publish_latest_loader_image.yml`](./.github/workflows/publish_latest_loader_image.yml) - [`.github/workflows/publish_release_loader_image.yml`](./.github/workflows/publish_release_loader_image.yml) - - [`.github/workflows/publish_latest_server_image.yml`](./.github/workflows/publish_latest_server_image.yml) - - [`.github/workflows/publish_release_server_image.yml`](./.github/workflows/publish_release_server_image.yml) - [`.github/workflows/publish_latest_hubble_image.yml`](./.github/workflows/publish_latest_hubble_image.yml) - [`.github/workflows/publish_release_hubble_image.yml`](./.github/workflows/publish_release_hubble_image.yml) - [`.github/workflows/publish_latest_vermeer_image.yml`](./.github/workflows/publish_latest_vermeer_image.yml) @@ -125,14 +129,18 @@ For example, [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.g - [`.github/workflows/publish_latest_ai_image.yml`](./.github/workflows/publish_latest_ai_image.yml) - [`.github/workflows/publish_release_ai_image.yml`](./.github/workflows/publish_release_ai_image.yml) -- Legacy or special-case workflows: +- Dedicated reusable publish path: + - [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) + - [`.github/workflows/publish_release_pd_store_server_image.yml`](./.github/workflows/publish_release_pd_store_server_image.yml) + +- Other legacy or special-case workflows: - [`.github/workflows/publish_hugegraph_hubble.yml`](./.github/workflows/publish_hugegraph_hubble.yml) - [`.github/workflows/publish_computer_image.yml`](./.github/workflows/publish_computer_image.yml) - - [`.github/workflows/publish_latest_pd_store_server_image.yml`](./.github/workflows/publish_latest_pd_store_server_image.yml) ## Practical Notes - `latest` workflows typically run on a schedule and accept manual dispatch. - `release` workflows typically accept only manual dispatch with a branch input. -- Most image workflows inherit credentials and settings through the reusable workflow. -- If you change the shared publishing behavior, update `_publish_image_reusable.yml` first and then adjust wrappers only where their inputs change. +- Most image workflows inherit credentials and settings through a reusable workflow. +- If you change shared standard behavior, update `_publish_image_reusable.yml` first. +- If you change pd/store/server behavior, update `_publish_pd_store_server_reusable.yml` first.