From c14827ce7700aae73787e765619ffb79e9f95912 Mon Sep 17 00:00:00 2001 From: Min Zhu Date: Thu, 11 Jun 2026 12:28:26 -0400 Subject: [PATCH 1/2] chore: remove hermetic build code generation workflow --- .github/scripts/update_generation_config.sh | 187 ------------------ .../hermetic_library_generation.yaml | 51 ----- .../workflows/update_generation_config.yaml | 43 ---- 3 files changed, 281 deletions(-) delete mode 100755 .github/scripts/update_generation_config.sh delete mode 100644 .github/workflows/hermetic_library_generation.yaml delete mode 100644 .github/workflows/update_generation_config.yaml diff --git a/.github/scripts/update_generation_config.sh b/.github/scripts/update_generation_config.sh deleted file mode 100755 index 3f7f46c1c257..000000000000 --- a/.github/scripts/update_generation_config.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/bash -set -ex -# This script should be run at the root of the repository. -# This script is used to update googleapis_commitish, gapic_generator_version, -# and libraries_bom_version in generation configuration at the time of running -# and create a pull request. - -# The following commands need to be installed before running the script: -# 1. git -# 2. gh -# 3. jq - -# Utility functions -# Get the latest released version of a Maven artifact. -function get_latest_released_version() { - local group_id=$1 - local artifact_id=$2 - group_id_url_path="$(sed 's|\.|/|g' <<< "${group_id}")" - url="https://repo1.maven.org/maven2/${group_id_url_path}/${artifact_id}/maven-metadata.xml" - xml_content=$(curl -s --fail "${url}") - - # 1. Extract all version tags - # 2. Strip the XML tags to leave just the version numbers - # 3. Filter for strictly numbers.numbers.numbers (e.g., 2.54.0) - # 4. Sort by version (V) and take the last one (tail -n 1) - latest=$(echo "${xml_content}" \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ - | sed -E 's/<[^>]+>//g' \ - | sort -V \ - | tail -n 1) - - if [[ -z "${latest}" ]]; then - echo "The latest version of ${group_id}:${artifact_id} is empty." - echo "The returned json from maven.org is invalid: ${json_content}" - exit 1 - else - echo "${latest}" - fi -} - -# Update a key to a new value in the generation config. -function update_config() { - local key_word=$1 - local new_value=$2 - local file=$3 - echo "Update ${key_word} to ${new_value} in ${file}" - sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" -} - -# Update an action to a new version in GitHub action. -function update_action() { - local key_word=$1 - local new_value=$2 - local file=$3 - echo "Update ${key_word} to ${new_value} in ${file}" - # use a different delimiter because the key_word contains "/". - sed -i -e "s|${key_word}@v.*$|${key_word}@v${new_value}|" "${file}" -} - -# The parameters of this script is: -# 1. base_branch, the base branch of the result pull request. -# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java -# 3. [optional] generation_config, the path to the generation configuration, -# the default value is generation_config.yaml in the repository root. -# 4. [optional] workflow, the library generation workflow file, -# the default value is .github/workflows/hermetic_library_generation.yaml. -while [[ $# -gt 0 ]]; do -key="$1" -case "${key}" in - --base_branch) - base_branch="$2" - shift - ;; - --repo) - repo="$2" - shift - ;; - --generation_config) - generation_config="$2" - shift - ;; - --workflow) - workflow="$2" - shift - ;; - *) - echo "Invalid option: [$1]" - exit 1 - ;; -esac -shift -done - -if [ -z "${base_branch}" ]; then - echo "missing required argument --base_branch" - exit 1 -fi - -if [ -z "${repo}" ]; then - echo "missing required argument --repo" - exit 1 -fi - -if [ -z "${generation_config}" ]; then - generation_config="generation_config.yaml" - echo "Use default generation config: ${generation_config}" -fi - -if [ -z "${workflow}" ]; then - workflow=".github/workflows/hermetic_library_generation.yaml" - echo "Use default library generation workflow file: ${workflow}" -fi - -current_branch="generate-libraries-${base_branch}" -title="chore: Update generation configuration at $(date)" - -git checkout "${base_branch}" -# Try to find a open pull request associated with the branch -pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") -# Create a branch if there's no open pull request associated with the -# branch; otherwise checkout the pull request. -if [ -z "${pr_num}" ]; then - git checkout -b "${current_branch}" - # Push the current branch to remote so that we can - # compare the commits later. - git push -u origin "${current_branch}" -else - gh pr checkout "${pr_num}" -fi - -# Only allow fast-forward merging; exit with non-zero result if there's merging -# conflict. -git merge -m "chore: merge ${base_branch} into ${current_branch}" "${base_branch}" - -mkdir tmp-googleapis -# Use partial clone because only commit history is needed. -git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis -pushd tmp-googleapis -git pull -latest_commit=$(git rev-parse HEAD) -popd -rm -rf tmp-googleapis -update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" - -# Update gapic-generator-java version to the latest -latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") -update_config "gapic_generator_version" "${latest_version}" "${generation_config}" - -# Update composite action version to latest gapic-generator-java version -update_action "googleapis/sdk-platform-java/.github/scripts" \ - "${latest_version}" \ - "${workflow}" - -# Update libraries-bom version to the latest -latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") -update_config "libraries_bom_version" "${latest_version}" "${generation_config}" - -git add "${generation_config}" "${workflow}" -changed_files=$(git diff --cached --name-only) -if [[ "${changed_files}" == "" ]]; then - echo "The latest generation config is not changed." - echo "Skip committing to the pull request." -else - git commit -m "${title}" -fi - -# There are potentially at most two commits: merge commit and change commit. -# We want to exit the script if no commit happens (otherwise this will be an -# infinite loop). -# `git cherry` is a way to find whether the local branch has commits that are -# not in the remote branch. -# If we find any such commit, push them to remote branch. -unpushed_commit=$(git cherry -v "origin/${current_branch}" | wc -l) -if [[ "${unpushed_commit}" -eq 0 ]]; then - echo "No unpushed commits, exit" - exit 0 -fi - -if [ -z "${pr_num}" ]; then - git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" - git fetch -q remote_repo - git push -f remote_repo "${current_branch}" - gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" -else - git push - gh pr edit "${pr_num}" --title "${title}" --body "${title}" -fi diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml deleted file mode 100644 index 10852d896d3b..000000000000 --- a/.github/workflows/hermetic_library_generation.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# GitHub action job to test core java library features on -# downstream client libraries before they are released. -name: Hermetic library generation upon generation config change through pull requests -on: - pull_request: - -env: - REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name }} - GITHUB_REPOSITORY: ${{ github.repository }} - # {x-version-update-start:gapic-generator-java:current} - GENERATOR_VERSION: 2.73.0 - # {x-version-update-end} -jobs: - library_generation: - runs-on: ubuntu-latest - steps: - - name: Determine whether the pull request comes from a fork - run: | - if [[ "${GITHUB_REPOSITORY}" != "${REPO_FULL_NAME}" ]]; then - echo "This PR comes from a fork. Skip library generation." - echo "SHOULD_RUN=false" >> $GITHUB_ENV - else - echo "SHOULD_RUN=true" >> $GITHUB_ENV - fi - - uses: actions/checkout@v4 - if: env.SHOULD_RUN == 'true' - with: - fetch-depth: 0 - token: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }} - - uses: ./sdk-platform-java/.github/scripts - if: env.SHOULD_RUN == 'true' - with: - base_ref: ${{ github.base_ref }} - head_ref: ${{ github.head_ref }} - token: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }} - force_regenerate_all: ${{ github.event.pull_request.head.ref == 'generate-libraries-main' }} - showcase_mode: true - image_tag: ${{ env.GENERATOR_VERSION }} diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml deleted file mode 100644 index c2c70d3f486c..000000000000 --- a/.github/workflows/update_generation_config.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# GitHub action job to test core java library features on -# downstream client libraries before they are released. -name: Update generation configuration -on: - workflow_dispatch: -jobs: - update-generation-config: - runs-on: ubuntu-24.04 - env: - # the branch into which the pull request is merged - base_branch: main - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }} - - name: Install Dependencies - shell: bash - run: sudo apt-get update && sudo apt-get install -y libxml2-utils - - name: Update params in generation config to latest - shell: bash - run: | - set -x - [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash .github/scripts/update_generation_config.sh \ - --base_branch "${base_branch}" \ - --repo ${{ github.repository }} - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }} From 518255a398decc703d5c11508ff800069574687b Mon Sep 17 00:00:00 2001 From: Min Zhu Date: Thu, 11 Jun 2026 13:39:42 -0400 Subject: [PATCH 2/2] revert rm of .github/scripts/update_generation_config.sh --- .github/scripts/update_generation_config.sh | 187 ++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100755 .github/scripts/update_generation_config.sh diff --git a/.github/scripts/update_generation_config.sh b/.github/scripts/update_generation_config.sh new file mode 100755 index 000000000000..3f7f46c1c257 --- /dev/null +++ b/.github/scripts/update_generation_config.sh @@ -0,0 +1,187 @@ +#!/bin/bash +set -ex +# This script should be run at the root of the repository. +# This script is used to update googleapis_commitish, gapic_generator_version, +# and libraries_bom_version in generation configuration at the time of running +# and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. jq + +# Utility functions +# Get the latest released version of a Maven artifact. +function get_latest_released_version() { + local group_id=$1 + local artifact_id=$2 + group_id_url_path="$(sed 's|\.|/|g' <<< "${group_id}")" + url="https://repo1.maven.org/maven2/${group_id_url_path}/${artifact_id}/maven-metadata.xml" + xml_content=$(curl -s --fail "${url}") + + # 1. Extract all version tags + # 2. Strip the XML tags to leave just the version numbers + # 3. Filter for strictly numbers.numbers.numbers (e.g., 2.54.0) + # 4. Sort by version (V) and take the last one (tail -n 1) + latest=$(echo "${xml_content}" \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + | sed -E 's/<[^>]+>//g' \ + | sort -V \ + | tail -n 1) + + if [[ -z "${latest}" ]]; then + echo "The latest version of ${group_id}:${artifact_id} is empty." + echo "The returned json from maven.org is invalid: ${json_content}" + exit 1 + else + echo "${latest}" + fi +} + +# Update a key to a new value in the generation config. +function update_config() { + local key_word=$1 + local new_value=$2 + local file=$3 + echo "Update ${key_word} to ${new_value} in ${file}" + sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" +} + +# Update an action to a new version in GitHub action. +function update_action() { + local key_word=$1 + local new_value=$2 + local file=$3 + echo "Update ${key_word} to ${new_value} in ${file}" + # use a different delimiter because the key_word contains "/". + sed -i -e "s|${key_word}@v.*$|${key_word}@v${new_value}|" "${file}" +} + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +# 4. [optional] workflow, the library generation workflow file, +# the default value is .github/workflows/hermetic_library_generation.yaml. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + --workflow) + workflow="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +if [ -z "${workflow}" ]; then + workflow=".github/workflows/hermetic_library_generation.yaml" + echo "Use default library generation workflow file: ${workflow}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: Update generation configuration at $(date)" + +git checkout "${base_branch}" +# Try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# Create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" + # Push the current branch to remote so that we can + # compare the commits later. + git push -u origin "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +# Only allow fast-forward merging; exit with non-zero result if there's merging +# conflict. +git merge -m "chore: merge ${base_branch} into ${current_branch}" "${base_branch}" + +mkdir tmp-googleapis +# Use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" + +# Update gapic-generator-java version to the latest +latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") +update_config "gapic_generator_version" "${latest_version}" "${generation_config}" + +# Update composite action version to latest gapic-generator-java version +update_action "googleapis/sdk-platform-java/.github/scripts" \ + "${latest_version}" \ + "${workflow}" + +# Update libraries-bom version to the latest +latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") +update_config "libraries_bom_version" "${latest_version}" "${generation_config}" + +git add "${generation_config}" "${workflow}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest generation config is not changed." + echo "Skip committing to the pull request." +else + git commit -m "${title}" +fi + +# There are potentially at most two commits: merge commit and change commit. +# We want to exit the script if no commit happens (otherwise this will be an +# infinite loop). +# `git cherry` is a way to find whether the local branch has commits that are +# not in the remote branch. +# If we find any such commit, push them to remote branch. +unpushed_commit=$(git cherry -v "origin/${current_branch}" | wc -l) +if [[ "${unpushed_commit}" -eq 0 ]]; then + echo "No unpushed commits, exit" + exit 0 +fi + +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push + gh pr edit "${pr_num}" --title "${title}" --body "${title}" +fi