diff --git a/.github/workflows/hermetic-build-scripts-ci.yaml b/.github/workflows/hermetic-build-scripts-ci.yaml deleted file mode 100644 index 19c4434980c6..000000000000 --- a/.github/workflows/hermetic-build-scripts-ci.yaml +++ /dev/null @@ -1,110 +0,0 @@ -on: - pull_request: - -name: hermetic build scripts -jobs: - filter: - runs-on: ubuntu-latest - outputs: - library: ${{ steps.filter.outputs.library }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 - id: filter - with: - filters: | - library: - - 'sdk-platform-java/hermetic_build/**' - - 'sdk-platform-java/.cloudbuild/library_generation/**' - - '.github/workflows/hermetic-build-scripts-ci.yaml' - library-generation-unit-tests: - needs: filter - if: needs.filter.outputs.library == 'true' - runs-on: ubuntu-22.04 - name: hermetic build units (python) - defaults: - run: - working-directory: sdk-platform-java - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: install python modules and dependencies - shell: bash - run: | - set -ex - pip install --upgrade pip - pip install --require-hashes -r hermetic_build/common/requirements.txt - pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - pip install hermetic_build/library_generation - pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt - pip install hermetic_build/release_note_generation - - name: Run shell unit tests - run: | - set -x - hermetic_build/library_generation/tests/generate_library_unit_tests.sh - - name: Run python unit tests - run: | - set -x - python -m unittest discover -s hermetic_build -p "*unit_tests.py" - library-generation-lint-shell: - needs: filter - if: needs.filter.outputs.library == 'true' - runs-on: ubuntu-22.04 - name: hermetic build lint (shell) - defaults: - run: - working-directory: sdk-platform-java - steps: - - uses: actions/checkout@v4 - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@2.0.0 - with: - scandir: 'sdk-platform-java/hermetic_build' - format: tty - severity: error - ignore_paths: - .kokoro - library-generation-lint-python: - needs: filter - if: needs.filter.outputs.library == 'true' - runs-on: ubuntu-22.04 - name: hermetic build lint (python) - defaults: - run: - working-directory: sdk-platform-java - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: install python dependencies - shell: bash - run: | - set -ex - pip install --upgrade pip - pip install --require-hashes -r hermetic_build/common/requirements.txt - pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - pip install hermetic_build/library_generation - pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt - pip install hermetic_build/release_note_generation - - name: Lint - shell: bash - run: | - # exclude generated golden files - # exclude owlbot until further refaction - black --check hermetic_build --exclude "(library_generation/tests/resources/goldens)" - required: - needs: [ library-generation-unit-tests, library-generation-lint-shell, library-generation-lint-python ] - name: conditional-required-check - if: ${{ always() }} # Always run even if any "needs" jobs fail - runs-on: ubuntu-22.04 - steps: - - name: Fail if any previous failure - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 - - name: Success otherwise - run: echo "Success!" \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index a3b8b988d09f..5a2b74edfeb6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,4 +58,3 @@ This is a standard Maven project. For development on core components, refer to the following guides in `sdk-platform-java`: - **GAPIC Generator**: [sdk-platform-java/gapic-generator-java/DEVELOPMENT.md](sdk-platform-java/gapic-generator-java/DEVELOPMENT.md) - **GAX**: [sdk-platform-java/gax-java/DEVELOPMENT.md](sdk-platform-java/gax-java/DEVELOPMENT.md) -- **Hermetic Build**: [sdk-platform-java/hermetic_build/DEVELOPMENT.md](sdk-platform-java/hermetic_build/DEVELOPMENT.md) diff --git a/release-please-config.json b/release-please-config.json index f9715f8e4a80..42dbb9043a87 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -10,14 +10,10 @@ "extra-files": [ "WORKSPACE", ".github/workflows/generated_files_sync.yaml", - ".github/workflows/hermetic_library_generation.yaml", "sdk-platform-java/.cloudbuild/graalvm/cloudbuild.yaml", "sdk-platform-java/.cloudbuild/graalvm/cloudbuild-test-a.yaml", "sdk-platform-java/.cloudbuild/graalvm/cloudbuild-test-b.yaml", "sdk-platform-java/.cloudbuild/graalvm/cloudbuild-test-c.yaml", - "sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml", - "sdk-platform-java/.cloudbuild/library_generation/library_generation.Dockerfile", - "sdk-platform-java/.cloudbuild/library_generation/library_generation_airlock.Dockerfile", "sdk-platform-java/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-native-a.cfg", "sdk-platform-java/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-native-b.cfg", "sdk-platform-java/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-native-c.cfg", diff --git a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml deleted file mode 100644 index 3bfe19506e1d..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -timeout: 7200s # 2 hours -substitutions: - _TEST_IMAGE: "test-image:latest" -steps: -- name: gcr.io/cloud-builders/docker - args: [ - "build", - "--no-cache", - # Use a fixed protoc version so that the generation result stays the same - "--build-arg", "PROTOC_VERSION=25.8", - "-t", "${_TEST_IMAGE}", - "-f", "sdk-platform-java/.cloudbuild/library_generation/library_generation_airlock.Dockerfile", - "." - ] - id: build-image - waitFor: ["-"] - env: - - "DOCKER_BUILDKIT=1" - -- name: alpine/git:latest - entrypoint: /bin/sh - args: - - "-c" - - | - cd /workspace - git clone https://github.com/googleapis/googleapis - cd googleapis - git checkout 113a378d5aad5018876ec0a8cbfd4d6a4f746809 - id: download-api-definitions - waitFor: ["-"] - -- name: alpine/git:latest - entrypoint: /bin/sh - args: - - "-c" - - | - cd /workspace - git clone https://github.com/googleapis/google-cloud-java - cd google-cloud-java - git checkout chore/test-hermetic-build - mkdir ../golden - cd ../golden - cp -r ../google-cloud-java/java-apigee-connect . - cp -r ../google-cloud-java/java-alloydb . - cp -r ../google-cloud-java/java-alloydb-connectors . - cp -r ../google-cloud-java/java-cloudcontrolspartner . - cp -r ../google-cloud-java/gapic-libraries-bom . - cp -r ../google-cloud-java/pom.xml . - id: prepare-golden - waitFor: ["-"] - -- name: maven:3.9.9-eclipse-temurin-11-alpine - entrypoint: /bin/sh - args: - - "-c" - - | - mvn dependency:copy \ - -B -ntp \ - -Dartifact=com.google.api:gapic-generator-java:2.63.0 \ - -DoutputDirectory=/workspace - cd /workspace - mv gapic-generator-java-2.63.0.jar gapic-generator-java.jar - id: prepare-generator-jar - waitFor: [ "-" ] - -- name: gcr.io/cloud-builders/docker - args: [ - "run", - "--rm", - "-v", "/workspace/google-cloud-java:/workspace", - "-v", "/workspace/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java:/workspace/config", - "-v", "/workspace/googleapis:/workspace/apis", - # Fix gapic-generator-java so that the generation result stays - # the same. - "-v", "/workspace/gapic-generator-java.jar:/home/.library_generation/gapic-generator-java.jar", - "${_TEST_IMAGE}", - "--generation-config-path=/workspace/config/generation_config.yaml", - "--api-definitions-path=/workspace/apis" - ] - env: - - "DOCKER_BUILDKIT=1" - id: generate-libraries - waitFor: [ - "build-image", - "download-api-definitions", - "prepare-golden", - "prepare-generator-jar" - ] - -- name: python:3.12.7-alpine3.20 - entrypoint: /bin/sh - args: - - "-c" - - | - python3 -m venv .venv - source .venv/bin/activate - pip install --require-hashes -r requirements.txt - python -m unittest integration_tests.py - dir: "sdk-platform-java/.cloudbuild/library_generation/scripts" - id: verify-generation - waitFor: ["generate-libraries"] -options: - logging: CLOUD_LOGGING_ONLY - machineType: E2_HIGHCPU_8 diff --git a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-push-exitgate.yaml b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-push-exitgate.yaml deleted file mode 100644 index d96f1dd99fef..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-push-exitgate.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -timeout: 7200s # 2 hours -substitutions: - _GAPIC_GENERATOR_JAVA_VERSION: '2.69.0' # {x-version-update:gapic-generator-java:current} - _IMAGE_NAME: "us-central1-docker.pkg.dev/cloud-sdk-production-pipeline/images-dev/java-library-generation" - _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}" - _VERSIONED_IMAGE_ID: "${_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}" -steps: - # Library generation build - - name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "${_SHA_IMAGE_ID}", - "-t", "${_VERSIONED_IMAGE_ID}", - "-f", "sdk-platform-java/.cloudbuild/library_generation/library_generation_airlock.Dockerfile", - "." - ] - id: library-generation-build - waitFor: ["-"] - env: - - 'DOCKER_BUILDKIT=1' - -options: - machineType: 'E2_HIGHCPU_8' - requestedVerifyOption: VERIFIED # For provenance attestation generation - -images: - - ${_SHA_IMAGE_ID} - - ${_VERSIONED_IMAGE_ID} diff --git a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml b/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml deleted file mode 100644 index 1e3fd3c63669..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -timeout: 7200s # 2 hours -substitutions: - _GAPIC_GENERATOR_JAVA_VERSION: '2.74.0-SNAPSHOT' # {x-version-update:gapic-generator-java:current} - _PRIVATE_IMAGE_NAME: "us-docker.pkg.dev/java-hermetic-build-prod/private-resources/java-library-generation" - _PRIVATE_SHA_IMAGE_ID: "${_PRIVATE_IMAGE_NAME}:${COMMIT_SHA}" - _PRIVATE_LATEST_IMAGE_ID: "${_PRIVATE_IMAGE_NAME}:latest" - _PRIVATE_VERSIONED_IMAGE_ID: "${_PRIVATE_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}" - _PUBLIC_IMAGE_NAME: "gcr.io/cloud-devrel-public-resources/java-library-generation" - _PUBLIC_SHA_IMAGE_ID: "${_PUBLIC_IMAGE_NAME}:infrastructure-public-image-${COMMIT_SHA}" - _PUBLIC_LATEST_IMAGE_ID: "${_PUBLIC_IMAGE_NAME}:latest" - _PUBLIC_VERSIONED_IMAGE_ID: "${_PUBLIC_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}" -steps: - # Library generation build - - name: gcr.io/cloud-builders/docker - args: [ - "build", - "-t", "${_PRIVATE_SHA_IMAGE_ID}", - "-t", "${_PRIVATE_LATEST_IMAGE_ID}", - "-t", "${_PRIVATE_VERSIONED_IMAGE_ID}", - "-t", "${_PUBLIC_SHA_IMAGE_ID}", - "-t", "${_PUBLIC_LATEST_IMAGE_ID}", - "-t", "${_PUBLIC_VERSIONED_IMAGE_ID}", - "-f", "sdk-platform-java/.cloudbuild/library_generation/library_generation_airlock.Dockerfile", - "." - ] - id: library-generation-build - waitFor: ["-"] - env: - - 'DOCKER_BUILDKIT=1' - -options: - logging: CLOUD_LOGGING_ONLY - -images: - - ${_PRIVATE_SHA_IMAGE_ID} - - ${_PRIVATE_LATEST_IMAGE_ID} - - ${_PRIVATE_VERSIONED_IMAGE_ID} - - ${_PUBLIC_SHA_IMAGE_ID} - - ${_PUBLIC_LATEST_IMAGE_ID} - - ${_PUBLIC_VERSIONED_IMAGE_ID} diff --git a/sdk-platform-java/.cloudbuild/library_generation/library_generation.Dockerfile b/sdk-platform-java/.cloudbuild/library_generation/library_generation.Dockerfile deleted file mode 100644 index 2b79df53de01..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/library_generation.Dockerfile +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install gapic-generator-java in a separate layer so we don't overload the image -# with the transferred source code and jars - - -FROM docker.io/library/maven:3.9.9-eclipse-temurin-17-alpine@sha256:969014ee8852c9910ff5ef09de17541c2587819364b79d7dc044634dfb8a3388 AS ggj-build - -WORKDIR /google-cloud-java - -COPY . . -# {x-version-update-start:gapic-generator-java:current} -ENV DOCKER_GAPIC_GENERATOR_VERSION="2.74.0-SNAPSHOT" -# {x-version-update-end} - -# Download the java formatter -RUN mvn -pl sdk-platform-java/gapic-generator-java-pom-parent help:evaluate -Dexpression='google-java-format.version' -q -DforceStdout > /java-formatter-version -RUN cat /java-formatter-version -RUN V=$(cat /java-formatter-version) && curl -o "/google-java-format.jar" "https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${V}/google-java-format-${V}-all-deps.jar" - -# Compile and install packages -RUN mvn install -B -ntp -T 1.5C -DskipTests -Dcheckstyle.skip -Dclirr.skip -Denforcer.skip -Dfmt.skip -pl sdk-platform-java/gapic-generator-java --also-make -RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \ - "/gapic-generator-java.jar" - -FROM docker.io/library/python:3.13.2-slim@sha256:6b3223eb4d93718828223966ad316909c39813dee3ee9395204940500792b740 as final - -ARG OWLBOT_CLI_COMMITTISH=3a68a9c0de318784b3aefadcc502a6521b3f1bc5 -ARG PROTOC_VERSION=33.2 -ARG GRPC_VERSION=1.81.0 -ENV HOME=/home -ENV OS_ARCHITECTURE="linux-x86_64" - -# {x-version-update-start:gapic-generator-java:current} -ENV GENERATOR_VERSION="2.74.0-SNAPSHOT" -# {x-version-update-end} - -# install OS tools -RUN apt update && apt install -y curl unzip rsync jq nodejs npm git openjdk-17-jdk - -SHELL [ "/bin/bash", "-c" ] - -# copy source code -COPY sdk-platform-java/hermetic_build/common /src/common -COPY sdk-platform-java/hermetic_build/library_generation /src/library_generation - -# install protoc -WORKDIR /protoc -RUN source /src/library_generation/utils/utilities.sh \ - && download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}" -# we indicate protoc is available in the container via env vars -ENV DOCKER_PROTOC_LOCATION=/protoc/bin -ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}" - -# install grpc -WORKDIR /grpc -RUN source /src/library_generation/utils/utilities.sh \ - && download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}" -# similar to protoc, we indicate grpc is available in the container via env vars -ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java.exe" - -RUN python -m pip install --upgrade pip - -# install main scripts as a python package -WORKDIR / -RUN python -m pip install --require-hashes -r src/common/requirements.txt -RUN python -m pip install src/common -RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt -RUN python -m pip install src/library_generation - -# install the owl-bot CLI -WORKDIR /tools -RUN git clone https://github.com/googleapis/repo-automation-bots -WORKDIR /tools/repo-automation-bots/packages/owl-bot -RUN git checkout "${OWLBOT_CLI_COMMITTISH}" -RUN npm i && npm run compile && npm link -RUN owl-bot copy-code --version -RUN chmod o+rx $(which owl-bot) - -# copy the Java formatter -COPY --from=ggj-build "/google-java-format.jar" "${HOME}"/.library_generation/google-java-format.jar -RUN chmod 755 "${HOME}"/.library_generation/google-java-format.jar -ENV JAVA_FORMATTER_LOCATION="${HOME}/.library_generation/google-java-format.jar" - -# Here we transfer gapic-generator-java from the previous stage. -# Note that the destination is a well-known location that will be assumed at runtime -# We hard-code the location string to avoid making it configurable (via ARG) as -# well as to avoid it making it overridable at runtime (via ENV). -COPY --from=ggj-build "/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar" -RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar" -ENV GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar" - -# allow users to access the script folders -RUN chmod -R o+rx /src - -# set dummy git credentials for the empty commit used in postprocessing -# we use system so all users using the container will use this configuration -RUN git config --system user.email "cloud-java-bot@google.com" -RUN git config --system user.name "Cloud Java Bot" - -# allow read-write for /home and execution for binaries in /home/.nvm -RUN chmod -R a+rw /home - -WORKDIR /workspace -ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ] diff --git a/sdk-platform-java/.cloudbuild/library_generation/library_generation_airlock.Dockerfile b/sdk-platform-java/.cloudbuild/library_generation/library_generation_airlock.Dockerfile deleted file mode 100644 index 2d4ad9e5ef56..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/library_generation_airlock.Dockerfile +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install gapic-generator-java in a separate layer so we don't overload the image -# with the transferred source code and jars - -# 3.9.9-eclipse-temurin-11-alpine -FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/maven@sha256:d3f04985c6a68415e36c0a6468d0f8316f27d4dbee77bc459257ba444224bd9f AS ggj-build - -WORKDIR /google-cloud-java - -COPY . . -# {x-version-update-start:gapic-generator-java:current} -ENV DOCKER_GAPIC_GENERATOR_VERSION="2.74.0-SNAPSHOT" -# {x-version-update-end} - -# Download the java formatter -RUN mvn -pl sdk-platform-java/gapic-generator-java-pom-parent help:evaluate -Dexpression='google-java-format.version' -q -DforceStdout > /java-formatter-version -RUN cat /java-formatter-version -RUN V=$(cat /java-formatter-version) && curl -o "/google-java-format.jar" "https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${V}/google-java-format-${V}-all-deps.jar" - -# Skipping the fmt check until 3.9.9-eclipse-temurin-17-alpine:969014ee8852 is available in Airlock. -RUN mvn install -B -ntp -T 1.5C -DskipTests -Dcheckstyle.skip -Dclirr.skip -Denforcer.skip -Dfmt.skip -pl sdk-platform-java/gapic-generator-java --also-make -RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \ - "/gapic-generator-java.jar" - -# 3.12.3-slim-bookworm -FROM us-docker.pkg.dev/artifact-foundry-prod/docker-3p-trusted/python@sha256:afc139a0a640942491ec481ad8dda10f2c5b753f5c969393b12480155fe15a63 as final - -ARG OWLBOT_CLI_COMMITTISH=3a68a9c0de318784b3aefadcc502a6521b3f1bc5 -ARG PROTOC_VERSION=33.2 -ARG GRPC_VERSION=1.81.0 -ENV HOME=/home -ENV OS_ARCHITECTURE="linux-x86_64" - -# install OS tools -RUN apt update && apt install -y curl unzip rsync jq nodejs npm git openjdk-17-jdk - -SHELL [ "/bin/bash", "-c" ] - -# copy source code -COPY sdk-platform-java/hermetic_build/common /src/common -COPY sdk-platform-java/hermetic_build/library_generation /src/library_generation - -# install protoc -WORKDIR /protoc -RUN source /src/library_generation/utils/utilities.sh \ - && download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}" -# we indicate protoc is available in the container via env vars -ENV DOCKER_PROTOC_LOCATION=/protoc/bin -ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}" - -# install grpc -WORKDIR /grpc -RUN source /src/library_generation/utils/utilities.sh \ - && download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}" -# similar to protoc, we indicate grpc is available in the container via env vars -ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java.exe" - -RUN python -m pip install --upgrade pip - -# install main scripts as a python package -WORKDIR / -RUN python -m pip install --require-hashes -r src/common/requirements.txt -RUN python -m pip install src/common -RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt -RUN python -m pip install src/library_generation - -# install the owl-bot CLI -WORKDIR /tools -RUN git clone https://github.com/googleapis/repo-automation-bots -WORKDIR /tools/repo-automation-bots/packages/owl-bot -RUN git checkout "${OWLBOT_CLI_COMMITTISH}" -RUN npm i && npm run compile && npm link -RUN owl-bot copy-code --version -RUN chmod o+rx $(which owl-bot) - -# copy the Java formatter -COPY --from=ggj-build "/google-java-format.jar" "${HOME}"/.library_generation/google-java-format.jar -RUN chmod 755 "${HOME}"/.library_generation/google-java-format.jar -ENV JAVA_FORMATTER_LOCATION="${HOME}/.library_generation/google-java-format.jar" - -# Here we transfer gapic-generator-java from the previous stage. -# Note that the destination is a well-known location that will be assumed at runtime -# We hard-code the location string to avoid making it configurable (via ARG) as -# well as to avoid it making it overridable at runtime (via ENV). -COPY --from=ggj-build "/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar" -RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar" -ENV GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar" - -# allow users to access the script folders -RUN chmod -R o+rx /src - -# set dummy git credentials for the empty commit used in postprocessing -# we use system so all users using the container will use this configuration -RUN git config --system user.email "cloud-java-bot@google.com" -RUN git config --system user.name "Cloud Java Bot" - -# allow read-write for /home and execution for binaries in /home/.nvm -RUN chmod -R a+rw /home - -WORKDIR /workspace -ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ] diff --git a/sdk-platform-java/.cloudbuild/library_generation/scripts/__init__.py b/sdk-platform-java/.cloudbuild/library_generation/scripts/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/.cloudbuild/library_generation/scripts/integration_tests.py b/sdk-platform-java/.cloudbuild/library_generation/scripts/integration_tests.py deleted file mode 100644 index f040fe84e59a..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/scripts/integration_tests.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import difflib -import json -import os -import sys -import unittest -import xml.etree.ElementTree as tree -from collections import Counter -from filecmp import dircmp - -script_dir = os.path.dirname(os.path.realpath(__file__)) - - -class IntegrationTest(unittest.TestCase): - - def test_monorepo_generation(self): - repo_dest = "/workspace/google-cloud-java" - golden_dir = "/workspace/golden" - library_names = [ - "java-apigee-connect", - "java-alloydb", - "java-alloydb-connectors", - "java-cloudcontrolspartner", - ] - for library_name in library_names: - actual_library = f"{repo_dest}/{library_name}" - print("*" * 50) - print(f"Checking for differences in '{library_name}'.") - print(f" The expected library is in {golden_dir}/{library_name}.") - print(f" The actual library is in {actual_library}.") - compare_result = dircmp( - f"{golden_dir}/{library_name}", - actual_library, - # README.md is ignored because the golden files in the - # google-cloud-java repository still contain the old build badges. - # TODO: Remove README.md from ignore list once golden files are updated. - ignore=[".repo-metadata.json", "README.md"], - ) - diff_files = [] - golden_only = [] - generated_only = [] - # compare source code - self.__recursive_diff_files( - compare_result, diff_files, golden_only, generated_only - ) - - # print all found differences for inspection - def print_file(f: str) -> None: - return print(f" - {f}") - - if len(diff_files) > 0: - print(" Some files (found in both folders) are differing:") - for diff_file in diff_files: - print(f"Difference in {diff_file}:") - with open( - f"{golden_dir}/{library_name}/{diff_file}" - ) as expected_file: - with open(f"{actual_library}/{diff_file}") as actual_file: - [ - print(line) - for line in difflib.unified_diff( - expected_file.readlines(), - actual_file.readlines(), - ) - ] - if len(golden_only) > 0: - print(" There were files found only in the golden dir:") - [print_file(f) for f in golden_only] - if len(generated_only) > 0: - print(" There were files found only in the generated dir:") - [print_file(f) for f in generated_only] - - self.assertTrue(len(golden_only) == 0) - self.assertTrue(len(generated_only) == 0) - self.assertTrue(len(diff_files) == 0) - - print(f" No differences found in {library_name}") - # compare .repo-metadata.json - self.assertTrue( - self.__compare_json_files( - f"{golden_dir}/{library_name}/.repo-metadata.json", - f"{actual_library}/.repo-metadata.json", - ), - msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", - ) - print(" .repo-metadata.json comparison succeed.") - # compare gapic-libraries-bom/pom.xml and pom.xml - self.assertFalse( - self.compare_xml( - f"{golden_dir}/gapic-libraries-bom/pom.xml", - f"{repo_dest}/gapic-libraries-bom/pom.xml", - False, - ) - ) - print(" gapic-libraries-bom/pom.xml comparison succeed.") - self.assertFalse( - self.compare_xml( - f"{golden_dir}/pom.xml", - f"{repo_dest}/pom.xml", - False, - ) - ) - print(" pom.xml comparison succeed.") - - @classmethod - def __compare_json_files(cls, expected: str, actual: str) -> bool: - return cls.__load_json_to_sorted_list( - expected - ) == cls.__load_json_to_sorted_list(actual) - - @classmethod - def __load_json_to_sorted_list(cls, path: str) -> list[tuple]: - with open(path) as f: - data = json.load(f) - res = [(key, value) for key, value in data.items()] - - return sorted(res, key=lambda x: x[0]) - - @classmethod - def __recursive_diff_files( - cls, - dcmp: dircmp, - diff_files: list[str], - left_only: list[str], - right_only: list[str], - dirname: str = "", - ): - """ - Recursively compares two subdirectories. The found differences are - passed to three expected list references. - """ - - def append_dirname(d: str) -> str: - return dirname + d - - diff_files.extend(map(append_dirname, dcmp.diff_files)) - left_only.extend(map(append_dirname, dcmp.left_only)) - right_only.extend(map(append_dirname, dcmp.right_only)) - for sub_dirname, sub_dcmp in dcmp.subdirs.items(): - cls.__recursive_diff_files( - sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" - ) - - @classmethod - def compare_xml(cls, expected, actual, print_trees): - """ - compares two XMLs for content differences - the argument print_whole_trees determines if both trees should be printed - """ - try: - expected_tree = tree.parse(expected) - actual_tree = tree.parse(actual) - except tree.ParseError as e: - cls.eprint(f"Error parsing XML") - raise e - except FileNotFoundError as e: - cls.eprint(f"Error reading file") - raise e - - expected_elements = [] - actual_elements = [] - - cls.append_to_element_list(expected_tree.getroot(), "/", expected_elements) - cls.append_to_element_list(actual_tree.getroot(), "/", actual_elements) - - expected_counter = Counter(expected_elements) - actual_counter = Counter(actual_elements) - intersection = expected_counter & actual_counter - only_in_expected = expected_counter - intersection - only_in_actual = actual_counter - intersection - if print_trees: - cls.eprint("expected") - cls.print_counter(actual_counter) - cls.eprint("actual") - cls.print_counter(expected_counter) - if len(only_in_expected) > 0 or len(only_in_actual) > 0: - cls.eprint("only in " + expected) - cls.print_counter(only_in_expected) - cls.eprint("only in " + actual) - cls.print_counter(only_in_actual) - return True - return False - - @classmethod - def append_to_element_list(cls, node, path, elements): - """ - Recursively traverses a node tree and appends element text to a given - `elements` array. If the element tag is `dependency` - then the maven coordinates for its children will be computed as well - """ - namespace_start, namespace_end, tag_name = node.tag.rpartition("}") - namespace = namespace_start + namespace_end - if tag_name == "dependency": - group_id = cls.get_text_from_element(node, "groupId", namespace) - artifact_id = cls.get_text_from_element(node, "artifactId", namespace) - artifact_str = "" - artifact_str += group_id - artifact_str += ":" + artifact_id - elements.append(path + "/" + tag_name + "=" + artifact_str) - if node.text and len(node.text.strip()) > 0: - elements.append(path + "/" + tag_name + "=" + node.text) - - if tag_name == "version": - # versions may be yet to be processed, we disregard them - return elements - - for child in node: - child_path = path + "/" + tag_name - cls.append_to_element_list(child, child_path, elements) - - return elements - - @classmethod - def get_text_from_element(cls, node, element_name, namespace): - """ - Convenience method to access a node's child elements via path and get - its text. - """ - child = node.find(namespace + element_name) - return child.text if child is not None else "" - - @classmethod - def eprint(cls, *args, **kwargs): - """ - prints to stderr - """ - print(*args, file=sys.stderr, **kwargs) - - @classmethod - def print_counter(cls, counter): - """ - Convenience method to pretty print the contents of a Counter (or dict) - """ - for key, value in counter.items(): - cls.eprint(f"{key}: {value}") diff --git a/sdk-platform-java/.cloudbuild/library_generation/scripts/requirements.in b/sdk-platform-java/.cloudbuild/library_generation/scripts/requirements.in deleted file mode 100644 index 32f2c45f3ab0..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/scripts/requirements.in +++ /dev/null @@ -1 +0,0 @@ -lxml==6.1.0 \ No newline at end of file diff --git a/sdk-platform-java/.cloudbuild/library_generation/scripts/requirements.txt b/sdk-platform-java/.cloudbuild/library_generation/scripts/requirements.txt deleted file mode 100644 index ae2024b22a45..000000000000 --- a/sdk-platform-java/.cloudbuild/library_generation/scripts/requirements.txt +++ /dev/null @@ -1,276 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --generate-hashes .cloudbuild/library_generation/scripts/requirements.in -# -lxml==6.1.0 \ - --hash=sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2 \ - --hash=sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2 \ - --hash=sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773 \ - --hash=sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773 \ - --hash=sha256:045e387d1f4f42a418380930fa3f45c73c9b392faf67e495e58902e68e8f44a7 \ - --hash=sha256:045e387d1f4f42a418380930fa3f45c73c9b392faf67e495e58902e68e8f44a7 \ - --hash=sha256:05b9b8787e35bec69e68daf4952b2e6dfcfb0db7ecf1a06f8cdfbbac4eb71aad \ - --hash=sha256:05b9b8787e35bec69e68daf4952b2e6dfcfb0db7ecf1a06f8cdfbbac4eb71aad \ - --hash=sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635 \ - --hash=sha256:07f98f5496f96bf724b1e3c933c107f0cbf2745db18c03d2e13a291c3afd2635 \ - --hash=sha256:08950a23f296b3f83521577274e3d3b0f3d739bf2e68d01a752e4288bc50d286 \ - --hash=sha256:08950a23f296b3f83521577274e3d3b0f3d739bf2e68d01a752e4288bc50d286 \ - --hash=sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9 \ - --hash=sha256:0d082495c5fcf426e425a6e28daaba1fcb6d8f854a4ff01effb1f1f381203eb9 \ - --hash=sha256:0f0f08beb0182e3e9a86fae124b3c47a7b41b7b69b225e1377db983802404e54 \ - --hash=sha256:0f0f08beb0182e3e9a86fae124b3c47a7b41b7b69b225e1377db983802404e54 \ - --hash=sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c \ - --hash=sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c \ - --hash=sha256:11a873c77a181b4fef9c2e357d08ed399542c2af1390101da66720a19c7c9618 \ - --hash=sha256:11a873c77a181b4fef9c2e357d08ed399542c2af1390101da66720a19c7c9618 \ - --hash=sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7 \ - --hash=sha256:183bfb45a493081943be7ea2b5adfc2b611e1cf377cefa8b8a8be404f45ef9a7 \ - --hash=sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39 \ - --hash=sha256:19f4164243fc206d12ed3d866e80e74f5bc3627966520da1a5f97e42c32a3f39 \ - --hash=sha256:1ae225f66e5938f4fa29d37e009a3bb3b13032ac57eb4eb42afa44f6e4054e69 \ - --hash=sha256:1ae225f66e5938f4fa29d37e009a3bb3b13032ac57eb4eb42afa44f6e4054e69 \ - --hash=sha256:1bc4cc83fb7f66ffb16f74d6dd0162e144333fc36ebcce32246f80c8735b2551 \ - --hash=sha256:1bc4cc83fb7f66ffb16f74d6dd0162e144333fc36ebcce32246f80c8735b2551 \ - --hash=sha256:1dd6a1c3ad4cb674f44525d9957f3e9c209bb6dd9213245195167a281fcc2bdc \ - --hash=sha256:1dd6a1c3ad4cb674f44525d9957f3e9c209bb6dd9213245195167a281fcc2bdc \ - --hash=sha256:20cf4d0651987c906a2f5cba4e3a8d6ba4bfdf973cfe2a96c0d6053888ea2ecd \ - --hash=sha256:20cf4d0651987c906a2f5cba4e3a8d6ba4bfdf973cfe2a96c0d6053888ea2ecd \ - --hash=sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd \ - --hash=sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd \ - --hash=sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405 \ - --hash=sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405 \ - --hash=sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289 \ - --hash=sha256:23a5dc68e08ed13331d61815c08f260f46b4a60fdd1640bbeb82cf89a9d90289 \ - --hash=sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b \ - --hash=sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b \ - --hash=sha256:2593a0a6621545b9095b71ad74ed4226eba438a7d9fc3712a99bdb15508cf93a \ - --hash=sha256:2593a0a6621545b9095b71ad74ed4226eba438a7d9fc3712a99bdb15508cf93a \ - --hash=sha256:264c605ab9c0e4aa1a679636f4582c4d3313700009fac3ec9c3412ed0d8f3e1d \ - --hash=sha256:264c605ab9c0e4aa1a679636f4582c4d3313700009fac3ec9c3412ed0d8f3e1d \ - --hash=sha256:26c5272c6a4bf4cf32d3f5a7890c942b0e04438691157d341616d02cca74d4bd \ - --hash=sha256:26c5272c6a4bf4cf32d3f5a7890c942b0e04438691157d341616d02cca74d4bd \ - --hash=sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac \ - --hash=sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac \ - --hash=sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62 \ - --hash=sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62 \ - --hash=sha256:29f5c00cb7d752bce2c70ebd2d31b0a42f9499ffdd3ecb2f31a5b73ee43031ad \ - --hash=sha256:29f5c00cb7d752bce2c70ebd2d31b0a42f9499ffdd3ecb2f31a5b73ee43031ad \ - --hash=sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f \ - --hash=sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f \ - --hash=sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad \ - --hash=sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad \ - --hash=sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c \ - --hash=sha256:363e47283bde87051b821826e71dde47f107e08614e1aa312ba0c5711e77738c \ - --hash=sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292 \ - --hash=sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292 \ - --hash=sha256:37448bf9c7d7adfc5254763901e2bbd6bb876228dfc1fc7f66e58c06368a7544 \ - --hash=sha256:37448bf9c7d7adfc5254763901e2bbd6bb876228dfc1fc7f66e58c06368a7544 \ - --hash=sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491 \ - --hash=sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491 \ - --hash=sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120 \ - --hash=sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120 \ - --hash=sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512 \ - --hash=sha256:3ae5d8d5427f3cc317e7950f2da7ad276df0cfa37b8de2f5658959e618ea8512 \ - --hash=sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb \ - --hash=sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb \ - --hash=sha256:40d9189f80075f2e1f88db21ef815a2b17b28adf8e50aaf5c789bfe737027f32 \ - --hash=sha256:40d9189f80075f2e1f88db21ef815a2b17b28adf8e50aaf5c789bfe737027f32 \ - --hash=sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2 \ - --hash=sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2 \ - --hash=sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec \ - --hash=sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec \ - --hash=sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88 \ - --hash=sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88 \ - --hash=sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43 \ - --hash=sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43 \ - --hash=sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037 \ - --hash=sha256:4642e04449a1e164b5ff71ffd901ddb772dfabf5c9adf1b7be5dffe1212bc037 \ - --hash=sha256:468479e52ecf3ec23799c863336d02c05fc2f7ffd1a1424eeeb9a28d4eb69d13 \ - --hash=sha256:468479e52ecf3ec23799c863336d02c05fc2f7ffd1a1424eeeb9a28d4eb69d13 \ - --hash=sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03 \ - --hash=sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03 \ - --hash=sha256:481d6e2104285d9add34f41b42b247b76b61c5b5c26c303c2e9707bbf8bd9a64 \ - --hash=sha256:481d6e2104285d9add34f41b42b247b76b61c5b5c26c303c2e9707bbf8bd9a64 \ - --hash=sha256:4937460dc5df0cdd2f06a86c285c28afda06aefa3af949f9477d3e8df430c485 \ - --hash=sha256:4937460dc5df0cdd2f06a86c285c28afda06aefa3af949f9477d3e8df430c485 \ - --hash=sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16 \ - --hash=sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16 \ - --hash=sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb \ - --hash=sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb \ - --hash=sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9 \ - --hash=sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9 \ - --hash=sha256:4e2c54d6b47361d0f1d3bc8d4e082ad87201e56ccdcca4d3b9ee3644ff595ec8 \ - --hash=sha256:4e2c54d6b47361d0f1d3bc8d4e082ad87201e56ccdcca4d3b9ee3644ff595ec8 \ - --hash=sha256:52b0ac6903cf74ebf997eb8c682d2fbac7d1ab7e4c552413eec55868a9b73f39 \ - --hash=sha256:52b0ac6903cf74ebf997eb8c682d2fbac7d1ab7e4c552413eec55868a9b73f39 \ - --hash=sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842 \ - --hash=sha256:546b66c0dd1bb8d9fa89d7123e5fa19a8aff3a1f2141eb22df96112afb17b842 \ - --hash=sha256:56971379bc5ee8037c5a0f09fa88f66cdb7d37c3e38af3e45cf539f41131ac1f \ - --hash=sha256:56971379bc5ee8037c5a0f09fa88f66cdb7d37c3e38af3e45cf539f41131ac1f \ - --hash=sha256:5715e0e28736a070f3f34a7ccc09e2fdcba0e3060abbcf61a1a5718ff6d6b105 \ - --hash=sha256:5715e0e28736a070f3f34a7ccc09e2fdcba0e3060abbcf61a1a5718ff6d6b105 \ - --hash=sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c \ - --hash=sha256:5cfa1a34df366d9dc0d5eaf420f4cf2bb1e1bebe1066d1c2fc28c179f8a4004c \ - --hash=sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7 \ - --hash=sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7 \ - --hash=sha256:6262b87f9e5c1e5fe501d6c153247289af42eb44ad7660b9b3de17baaf92d6f6 \ - --hash=sha256:6262b87f9e5c1e5fe501d6c153247289af42eb44ad7660b9b3de17baaf92d6f6 \ - --hash=sha256:63aeafc26aac0be8aff14af7871249e87ea1319be92090bfd632ec68e03b16a5 \ - --hash=sha256:63aeafc26aac0be8aff14af7871249e87ea1319be92090bfd632ec68e03b16a5 \ - --hash=sha256:690022c7fae793b0489aa68a658822cea83e0d5933781811cabbf5ea3bcfe73d \ - --hash=sha256:690022c7fae793b0489aa68a658822cea83e0d5933781811cabbf5ea3bcfe73d \ - --hash=sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9 \ - --hash=sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9 \ - --hash=sha256:73becf6d8c81d4c76b1014dbd3584cb26d904492dcf73ca85dc8bff08dcd6d2d \ - --hash=sha256:73becf6d8c81d4c76b1014dbd3584cb26d904492dcf73ca85dc8bff08dcd6d2d \ - --hash=sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb \ - --hash=sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb \ - --hash=sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f \ - --hash=sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f \ - --hash=sha256:76b958b4ea3104483c20f74866d55aa056546e15ebe83dd7aecd63698f43b755 \ - --hash=sha256:76b958b4ea3104483c20f74866d55aa056546e15ebe83dd7aecd63698f43b755 \ - --hash=sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb \ - --hash=sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb \ - --hash=sha256:7ba11752e346bd804ea312ec2eea2532dfa8b8d3261d81a32ef9e6ab16256280 \ - --hash=sha256:7ba11752e346bd804ea312ec2eea2532dfa8b8d3261d81a32ef9e6ab16256280 \ - --hash=sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace \ - --hash=sha256:7da13bb6fbadfafb474e0226a30570a3445cfd47c86296f2446dafbd77079ace \ - --hash=sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e \ - --hash=sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e \ - --hash=sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33 \ - --hash=sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33 \ - --hash=sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5 \ - --hash=sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5 \ - --hash=sha256:81ff55c70b67d19d52b6fd118a114c0a4c97d799cd3089ff9bd9e2ff4b414ee2 \ - --hash=sha256:81ff55c70b67d19d52b6fd118a114c0a4c97d799cd3089ff9bd9e2ff4b414ee2 \ - --hash=sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3 \ - --hash=sha256:857efde87d365706590847b916baff69c0bc9252dc5af030e378c9800c0b10e3 \ - --hash=sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585 \ - --hash=sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585 \ - --hash=sha256:8c11b984b5ce6add4dccc7144c7be5d364d298f15b0c6a57da1991baedc750ce \ - --hash=sha256:8c11b984b5ce6add4dccc7144c7be5d364d298f15b0c6a57da1991baedc750ce \ - --hash=sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946 \ - --hash=sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946 \ - --hash=sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e \ - --hash=sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e \ - --hash=sha256:9147d8e386ec3b82c3b15d88927f734f565b0aaadef7def562b853adca45784a \ - --hash=sha256:9147d8e386ec3b82c3b15d88927f734f565b0aaadef7def562b853adca45784a \ - --hash=sha256:920354904d1cb86577d4b3cfe2830c2dbe81d6f4449e57ada428f1609b5985f7 \ - --hash=sha256:920354904d1cb86577d4b3cfe2830c2dbe81d6f4449e57ada428f1609b5985f7 \ - --hash=sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5 \ - --hash=sha256:942454ff253da14218f972b23dc72fa4edf6c943f37edd19cd697618b626fac5 \ - --hash=sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28 \ - --hash=sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28 \ - --hash=sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4 \ - --hash=sha256:976a6b39b1b13e8c354ad8d3f261f3a4ac6609518af91bdb5094760a08f132c4 \ - --hash=sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c \ - --hash=sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c \ - --hash=sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50 \ - --hash=sha256:9c03e048b6ce8e77b09c734e931584894ecd58d08296804ca2d0b184c933ce50 \ - --hash=sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9 \ - --hash=sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9 \ - --hash=sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495 \ - --hash=sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495 \ - --hash=sha256:9f93d5b8b07f73e8c77e3c6556a3db269918390c804b5e5fcdd4858232cc8f16 \ - --hash=sha256:9f93d5b8b07f73e8c77e3c6556a3db269918390c804b5e5fcdd4858232cc8f16 \ - --hash=sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45 \ - --hash=sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45 \ - --hash=sha256:a02ca8fe48815bddcfca3248efe54451abb9dbf2f7d1c5744c8aa4142d476919 \ - --hash=sha256:a02ca8fe48815bddcfca3248efe54451abb9dbf2f7d1c5744c8aa4142d476919 \ - --hash=sha256:a1d9b99e5b2597e4f5aed2484fef835256fa1b68a19e4265c97628ef4bf8bcf4 \ - --hash=sha256:a1d9b99e5b2597e4f5aed2484fef835256fa1b68a19e4265c97628ef4bf8bcf4 \ - --hash=sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc \ - --hash=sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc \ - --hash=sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec \ - --hash=sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec \ - --hash=sha256:a86f06f059e22a0d574990ee2df24ede03f7f3c68c1336293eee9536c4c776cd \ - --hash=sha256:a86f06f059e22a0d574990ee2df24ede03f7f3c68c1336293eee9536c4c776cd \ - --hash=sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8 \ - --hash=sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8 \ - --hash=sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f \ - --hash=sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f \ - --hash=sha256:b6c2f225662bc5ad416bdd06f72ca301b31b39ce4261f0e0097017fc2891b940 \ - --hash=sha256:b6c2f225662bc5ad416bdd06f72ca301b31b39ce4261f0e0097017fc2891b940 \ - --hash=sha256:bb40648d96157f9081886defe13eac99253e663be969ff938a9289eff6e47b72 \ - --hash=sha256:bb40648d96157f9081886defe13eac99253e663be969ff938a9289eff6e47b72 \ - --hash=sha256:bba078de0031c219e5dd06cf3e6bf8fb8e6e64a77819b358f53bb132e3e03366 \ - --hash=sha256:bba078de0031c219e5dd06cf3e6bf8fb8e6e64a77819b358f53bb132e3e03366 \ - --hash=sha256:bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814 \ - --hash=sha256:bc783ee3147e60a25aa0445ea82b3e8aabb83b240f2b95d32cb75587ff781814 \ - --hash=sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690 \ - --hash=sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690 \ - --hash=sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13 \ - --hash=sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13 \ - --hash=sha256:c08da09dc003c9e8c70e06b53a11db6fb3b250c21c4236b03c7d7b443c318e7a \ - --hash=sha256:c08da09dc003c9e8c70e06b53a11db6fb3b250c21c4236b03c7d7b443c318e7a \ - --hash=sha256:c3592631e652afa34999a088f98ba7dfc7d6aff0d535c410bea77a71743f3819 \ - --hash=sha256:c3592631e652afa34999a088f98ba7dfc7d6aff0d535c410bea77a71743f3819 \ - --hash=sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c \ - --hash=sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c \ - --hash=sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86 \ - --hash=sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86 \ - --hash=sha256:c53fa3a5a52122d590e847a57ccf955557b9634a7f99ff5a35131321b0a85317 \ - --hash=sha256:c53fa3a5a52122d590e847a57ccf955557b9634a7f99ff5a35131321b0a85317 \ - --hash=sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180 \ - --hash=sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180 \ - --hash=sha256:c748ebcb6877de89f48ab90ca96642ac458fff5dec291a2b9337cd4d0934e383 \ - --hash=sha256:c748ebcb6877de89f48ab90ca96642ac458fff5dec291a2b9337cd4d0934e383 \ - --hash=sha256:c871299c595ee004d186f61840f0bfc4941aa3f17c8ba4a565ead7e4f4f820ee \ - --hash=sha256:c871299c595ee004d186f61840f0bfc4941aa3f17c8ba4a565ead7e4f4f820ee \ - --hash=sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe \ - --hash=sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe \ - --hash=sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181 \ - --hash=sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181 \ - --hash=sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9 \ - --hash=sha256:cec05be8c876f92a5aa07b01d60bbb4d11cfbdd654cad0561c0d7b5c043a61b9 \ - --hash=sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e \ - --hash=sha256:d036ee7b99d5148072ac7c9b847193decdfeac633db350363f7bce4fff108f0e \ - --hash=sha256:d0d799ff958655781296ec870d5e2448e75150da2b3d07f13ff5b0c2c35beefd \ - --hash=sha256:d0d799ff958655781296ec870d5e2448e75150da2b3d07f13ff5b0c2c35beefd \ - --hash=sha256:d1392c569c032f78a11a25d1de1c43fff13294c793b39e19d84fade3045cbbc3 \ - --hash=sha256:d1392c569c032f78a11a25d1de1c43fff13294c793b39e19d84fade3045cbbc3 \ - --hash=sha256:d2f17a16cd8751e8eb233a7e41aecdf8e511712e00088bf9be455f604cd0d28d \ - --hash=sha256:d2f17a16cd8751e8eb233a7e41aecdf8e511712e00088bf9be455f604cd0d28d \ - --hash=sha256:d3829a6e6fd550a219564912d4002c537f65da4c6ae4e093cc34462f4fa027ad \ - --hash=sha256:d3829a6e6fd550a219564912d4002c537f65da4c6ae4e093cc34462f4fa027ad \ - --hash=sha256:d43aa26dcda363f21e79afa0668f5029ed7394b3bb8c92a6927a3d34e8b610ea \ - --hash=sha256:d43aa26dcda363f21e79afa0668f5029ed7394b3bb8c92a6927a3d34e8b610ea \ - --hash=sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24 \ - --hash=sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24 \ - --hash=sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d \ - --hash=sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d \ - --hash=sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de \ - --hash=sha256:db88156fcf544cdbf0d95588051515cfdfd4c876fc66444eb98bceb5d6db76de \ - --hash=sha256:de550d129f18d8ab819651ffe4f38b1b713c7e116707de3c0c6400d0ef34fbc1 \ - --hash=sha256:de550d129f18d8ab819651ffe4f38b1b713c7e116707de3c0c6400d0ef34fbc1 \ - --hash=sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d \ - --hash=sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d \ - --hash=sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11 \ - --hash=sha256:e3c4f84b24a1fcba435157d111c4b755099c6ff00a3daee1ad281817de75ed11 \ - --hash=sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9 \ - --hash=sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9 \ - --hash=sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2 \ - --hash=sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2 \ - --hash=sha256:e80807d72f96b96ad5588cb85c75616e4f2795a7737d4630784c51497beb7776 \ - --hash=sha256:e80807d72f96b96ad5588cb85c75616e4f2795a7737d4630784c51497beb7776 \ - --hash=sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f \ - --hash=sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f \ - --hash=sha256:f0cea5b1d3e6e77d71bd2b9972eb2446221a69dc52bb0b9c3c6f6e5700592d93 \ - --hash=sha256:f0cea5b1d3e6e77d71bd2b9972eb2446221a69dc52bb0b9c3c6f6e5700592d93 \ - --hash=sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a \ - --hash=sha256:f15401d8d3dbf239e23c818afc10c7207f7b95f9a307e092122b6f86dd43209a \ - --hash=sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5 \ - --hash=sha256:f504d861d9f2a8f94020130adac88d66de93841707a23a86244263d1e54682f5 \ - --hash=sha256:fc46da94826188ed45cb53bd8e3fc076ae22675aea2087843d4735627f867c6d \ - --hash=sha256:fc46da94826188ed45cb53bd8e3fc076ae22675aea2087843d4735627f867c6d \ - --hash=sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d \ - --hash=sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d \ - --hash=sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3 \ - --hash=sha256:fcf3da95e93349e0647d48d4b36a12783105bcc74cb0c416952f9988410846a3 \ - --hash=sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086 \ - --hash=sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086 \ - --hash=sha256:ffb34ea45a82dd637c2c97ae1bbb920850c1e59bcae79ce1c15af531d83e7215 \ - --hash=sha256:ffb34ea45a82dd637c2c97ae1bbb920850c1e59bcae79ce1c15af531d83e7215 - # via -r requirements.in diff --git a/sdk-platform-java/.github/scripts/action.yaml b/sdk-platform-java/.github/scripts/action.yaml deleted file mode 100644 index 7546ec0e1f4f..000000000000 --- a/sdk-platform-java/.github/scripts/action.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# GitHub action job to test core java library features on -# downstream client libraries before they are released. - -# This composite action should be used in google-cloud-java and handwritten -# libraries to generate changed libraries. -# This composite action serves as a source of truth of scripts that run -# library generation and create pull requests. -name: Hermetic library generation -description: Runs hermetic library generation to produce changed libraries -inputs: - base_ref: - description: base branch - required: true - head_ref: - description: head branch - required: true - image_tag: - description: the tag of hermetic build image - required: false - showcase_mode: - description: true if we need to download the showcase api definitions - required: false - token: - description: Personal Access Token - required: true - force_regenerate_all: - description: true if we want to regenerate all libraries - required: false - -runs: - using: "composite" - steps: - - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Copy python script - shell: bash - run: | - set -x - # repository root - cd ${{ github.action_path }}/../../ - rsync -rv \ - --exclude=tests \ - hermetic_build "${GITHUB_WORKSPACE}" - - name: Copy shell script - shell: bash - run: | - cd ${{ github.action_path }} - cp hermetic_library_generation.sh "${GITHUB_WORKSPACE}" - - name: Install python packages - shell: bash - run: | - cd "${GITHUB_WORKSPACE}" - pip install --require-hashes -r hermetic_build/common/requirements.txt - pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt - pip install hermetic_build/release_note_generation - - name: Generate changed libraries - shell: bash - run: | - set -x - [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - cd "${GITHUB_WORKSPACE}" - bash hermetic_library_generation.sh \ - --target_branch "${BASE_REF}" \ - --current_branch "${HEAD_REF}" \ - --showcase_mode "${SHOWCASE_MODE}" \ - --image_tag "${IMAGE_TAG}" \ - --force_regenerate_all "${FORCE_REGENERATE_ALL}" - env: - BASE_REF: ${{ inputs.base_ref }} - HEAD_REF: ${{ inputs.head_ref }} - IMAGE_TAG: ${{ inputs.image_tag }} - SHOWCASE_MODE: ${{ inputs.showcase_mode }} - GH_TOKEN: ${{ inputs.token }} - FORCE_REGENERATE_ALL: ${{ inputs.force_regenerate_all }} diff --git a/sdk-platform-java/.github/scripts/hermetic_library_generation.sh b/sdk-platform-java/.github/scripts/hermetic_library_generation.sh deleted file mode 100755 index c254aae64b3a..000000000000 --- a/sdk-platform-java/.github/scripts/hermetic_library_generation.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash -set -exo pipefail -# This script should be run at the root of the repository. -# This script is used to, when a pull request changes the generation -# configuration (generation_config.yaml by default) or Dockerfile: -# 1. Find whether the last commit in this pull request contains changes to -# the generation configuration and Dockerfile and exit early if it doesn't have -# such a change since the generation result would be the same. -# 2. Compare generation configurations in the current branch (with which the -# pull request associated) and target branch (into which the pull request is -# merged); -# 3. Generate changed libraries using library_generation image; -# 4. Commit the changes to the pull request, if any. -# 5. Edit the PR body with generated pull request description, if applicable. - -# The following commands need to be installed before running the script: -# 1. git -# 2. gh -# 3. docker -# 4. mvn - -# The parameters of this script is: -# 1. target_branch, the branch into which the pull request is merged. -# 2. current_branch, the branch with which the pull request is associated. -# 3. [optional] image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. -# 4. [optional] generation_config, the path to the generation configuration, -# the default value is generation_config.yaml in the repository root. -# 5. [optional] showcase_mode, true if we wish to download the showcase api -# definitions, which are necessary for generating the showcase library. -IMAGE_NAME="gcr.io/cloud-devrel-public-resources/java-library-generation" -while [[ $# -gt 0 ]]; do -key="$1" -case "${key}" in - --target_branch) - target_branch="$2" - shift - ;; - --current_branch) - current_branch="$2" - shift - ;; - --image_tag) - image_tag="$2" - shift - ;; - --generation_config) - generation_config="$2" - shift - ;; - --showcase_mode) - showcase_mode="$2" - shift - ;; - --force_regenerate_all) - force_regenerate_all="$2" - shift - ;; - *) - echo "Invalid option: [$1]" - exit 1 - ;; -esac -shift -done - -if [ -z "${target_branch}" ]; then - echo "missing required argument --target_branch" - exit 1 -fi - -if [ -z "${current_branch}" ]; then - echo "missing required argument --current_branch" - exit 1 -fi - -if [ -z "${download_showcase}" ]; then - download_showcase="false" -fi - -if [ -z "${generation_config}" ]; then - generation_config=generation_config.yaml - echo "Use default generation config: ${generation_config}" -fi - -if [ -z "${image_tag}" ]; then - image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs) -fi - -if [ -z "${force_regenerate_all}" ]; then - force_regenerate_all="false" -fi - -workspace_name="/workspace" -baseline_generation_config="baseline_generation_config.yaml" -message="chore: generate libraries at $(date)" - -git checkout "${target_branch}" -git checkout "${current_branch}" - -# copy generation configuration from target branch to current branch. -git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" - -# download api definitions from googleapis repository -googleapis_commitish=$(grep googleapis_commitish "${generation_config}" | cut -d ":" -f 2 | xargs) -api_def_dir=$(mktemp -d) -git clone https://github.com/googleapis/googleapis.git "${api_def_dir}" -pushd "${api_def_dir}" -git checkout "${googleapis_commitish}" -popd - -# we also setup showcase -if [[ "${showcase_mode}" == "true" ]]; then - source java-showcase/scripts/showcase_utilities.sh - append_showcase_to_api_defs "${api_def_dir}" -fi - -# get changed library list. -changed_libraries_file="$(mktemp)" -python hermetic_build/common/cli/get_changed_libraries.py create \ - --baseline-generation-config-path="${baseline_generation_config}" \ - --current-generation-config-path="${generation_config}"\ - --force-regenerate-all="${force_regenerate_all}" | tee "${changed_libraries_file}" -changed_libraries="$(cat "${changed_libraries_file}")" -echo "Changed libraries are: ${changed_libraries:-"No changed library"}." - -# run hermetic code generation docker image. -bash generation/run_generator_docker.sh "${image_tag}" "${target_branch}" \ - -u "$(id -u):$(id -g)" \ - -v "$(pwd):${workspace_name}" \ - -v "${api_def_dir}:${workspace_name}/googleapis" \ - -e GENERATOR_VERSION="${image_tag}" \ - -- \ - --generation-config-path="${workspace_name}/${generation_config}" \ - --library-names="${changed_libraries}" \ - --api-definitions-path="${workspace_name}/googleapis" - -python hermetic_build/release_note_generation/cli/generate_release_note.py generate \ - --baseline-generation-config-path="${baseline_generation_config}" \ - --current-generation-config-path="${generation_config}" - -# remove api definitions after generation -rm -rf "${api_def_dir}" - -# commit the change to the pull request. -rm -rdf output googleapis "${baseline_generation_config}" -git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh' ':!hermetic_build' -changed_files=$(git diff --cached --name-only) -if [[ "${changed_files}" != "" ]]; then - echo "Commit changes..." - git commit -m "${message}" - git push -else - echo "There is no generated code change, skip commit." -fi - -# set pr body if pr_description.txt is generated. -if [[ -f "pr_description.txt" ]]; then - pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") - gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" -fi diff --git a/sdk-platform-java/hermetic_build/DEVELOPMENT.md b/sdk-platform-java/hermetic_build/DEVELOPMENT.md deleted file mode 100644 index fc3ea2a72856..000000000000 --- a/sdk-platform-java/hermetic_build/DEVELOPMENT.md +++ /dev/null @@ -1,259 +0,0 @@ -> [!IMPORTANT] -> All examples assume you are inside the repository root folder. - - -# Linting - -When contributing, ensure your changes to python code have a valid format. - -``` -python -m pip install black -black {source_file_or_directory} -``` - -# Install package dependencies - -```shell -python -m pip install --require-hashes -r hermetic_build/common/requirements.txt -python -m pip install hermetic_build/common -python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt -python -m pip install hermetic_build/library_generation -python -m pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt -python -m pip install hermetic_build/release_note_generation -``` - -# Run the integration tests - -The integration tests build the docker image declared in -`.cloudbuild/library_generation/library_generation.Dockerfile`, pull API -definitions and GAPIC repositories, generate the libraries and compare the -results with the source code declared in a "golden branch" of the repo. - -The integration tests are running in Cloud Build rather than GitHub workflow -because the workflow doesn't have permission to pull images from Airlock. - -The Cloud Build job is defined in `.cloudbuild/library_generation/cloudbuild-library-generation-integration-test.yaml` and runs in every pull request. - -# Run the unit tests - -There is one unit test file per component. -Every unit test script ends with `unit_tests.py`. -To avoid specifying them individually, we can use the following command: - -```shell -python -m unittest discover -s hermetic_build -p "*unit_tests.py" -``` - -> [!NOTE] -> The output of this command may look erratic during the first 30 seconds. -> This is normal. After the tests are done, an "OK" message should be shown. - -# Run the library generation scripts in your local environment - -Although the scripts are designed to run in a Docker container, you can also -run them directly. -This section explains how to run the entrypoint script -(`hermetic_build/library_generation/cli/entry_point.py`). - -## Assumptions made by the scripts - -### The Hermetic Build's well-known folder -Located in `${HOME}/.library_generation`, this folder is assumed by the scripts -to contain certain tools. - -Developers must make sure this folder is properly configured before running the -scripts locally. -Note that this relies on the `HOME` environment variable which is always defined -as per [POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). - -#### Put the gapic-generator-java jar in its well-known location - -1. Run the following command to install gapic-generator-java. - - ```shell - mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip - ``` - This will generate a jar located in `~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` - -2. Move the jar into its well-known location. - - ```shell - mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" - ``` - -#### Put the protobuf compiler in its well-known location -1. Download protobuf compiler from [GitHub releases](https://github.com/protocolbuffers/protobuf/releases). -2. Move the folder into its well-know location. - - ```shell - unzip /path/to/zipfile -d "${HOME}/.library_generation/" - ``` -#### Put the gRPC plugin in its well-known location -1. Download gRPC plugin from [Maven Central](https://central.sonatype.com/artifact/io.grpc/protoc-gen-grpc-java/versions). -2. Move the folder into its well-know location. - - ```shell - mv /path/to/protoc-gen-grpc-java.exe "${HOME}/.library_generation/protoc-gen-grpc-java.exe" - ``` - -#### Put the java formatter jar in its well-known location - -1. Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format) -or [GitHub releases](https://github.com/google/google-java-format/releases). -2. Move the jar into its well-known location. - - ```shell - mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar" - ``` - -## Installing prerequisites - -In order to run the generation scripts directly, there are a few tools we -need to install beforehand. - -### Install the owl-bot CLI - -This requires node.js to be installed. -Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) -for NVM, Node.js's version manager. - -After you install it, you can install the owl-bot CLI with the following -commands: - -```shell -git clone https://github.com/googleapis/repo-automation-bots -cd repo-automation-bots/packages/owl-bot -npm i && npm run compile && npm link -owl-bot copy-code --version -``` - -The key step is `npm link`, which will make the command available in you current -shell session. - -If you get a permission denied error when running the command `owl-bot`, try -relinking owl-bot by running `npm unlink -g` and re-running the steps above. - -## Run the script -The entrypoint script (`hermetic_build/library_generation/cli/entry_point.py`) -allows you to generate a GAPIC repository with a given api definition (proto, -service yaml). - -### Download the api definition -For example, from googleapis - -```shell -git clone https://github.com/googleapis/googleapis -export api_definitions_path="$(pwd)/googleapis" -``` - -### Download the repo -For example, google-cloud-java -```shell -git clone https://github.com/googleapis/google-cloud-java -export path_to_repo="$(pwd)/google-cloud-java" -``` - -### Install the scripts - -You can skip this step if you've installed the packages in [Install package dependencies](#install-package-dependencies). -Use the `--editable` flag for your changes to take effect as soon as you modify any file inside -the package. - -```shell -python -m pip install --require-hashes -r hermetic_build/common/requirements.txt -python -m pip install --editable hermetic_build/common -python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt -python -m pip install --editable hermetic_build/library_generation -``` - -### Run the script - -```shell -python hermetic_build/library_generation/cli/entry_point.py generate \ - --repository-path="${path_to_repo}" \ - --api-definitions-path="${api_definitions_path}" -``` - -# Build the image from source - -1. Run the following command to build the image from source - - ```shell - DOCKER_BUILDKIT=1 docker build \ - -f .cloudbuild/library_generation/library_generation.Dockerfile \ - -t local:image-tag \ - . - ``` - Please note that the build only works when using the new [Docker BuildKit](https://docs.docker.com/build/buildkit/) - (enabled through the `DOCKER_BUILDKIT` variable). - -2. Set the version of gapic-generator-java - - ```shell - export LOCAL_GENERATOR_VERSION=$(mvn \ - org.apache.maven.plugins:maven-help-plugin:evaluate \ - -Dexpression=project.version \ - -pl sdk-platform-java/gapic-generator-java \ - -DforceStdout \ - -q) - ``` - -3. Clone the googleapis repository (API definitions) - ```shell - cd google-cloud-java - git clone https://github.com/googleapis/googleapis - ``` - -4. Run the image - - ```shell - # Assume you want to generate the library in the current working directory - # and the generation configuration is in the same directory. - docker run \ - --rm \ - --quiet \ - -u "$(id -u):$(id -g)" \ - -v "$(pwd):/workspace" \ - -v "$(pwd)/googleapis:/googleapis" \ - -e GENERATOR_VERSION="${LOCAL_GENERATOR_VERSION}" \ - local:image-tag \ - --generation-config-path=/workspace/generation_config.yaml \ - --library-names=translate \ - --repository-path=/workspace \ - --api-definitions-path=/googleapis - ``` - -# Debugging tips -## Debug the scripts -Especially on local setups, consider adding the `xtrace` (`set -x`) flag to - - `hermetic_build/library_generation/generate_library.sh` - - `hermetic_build/library_generation/utils/utilities.sh` - -This will allow you to observe how the tools you prepared in `~/.library_generation` are being used. - - -## Debug the library generation container -If you are working on changing the way the containers are created, you may want -to inspect the containers to check the setup. -It would be convenient in such case to have a text editor/viewer available. -You can achieve this by modifying the Dockerfile as follows: - -```dockerfile -# install OS tools -RUN apk update && apk add \ - unzip curl rsync openjdk11 jq bash nodejs npm git less vim -``` - -We add `less` and `vim` as text tools for further inspection. - -You can also run a shell in a new container by running: - -```shell -docker run \ - --rm \ - -it \ - -u $(id -u):$(id -g) \ - -v /path/to/google-cloud-java:/workspace \ - --entrypoint="bash" \ - $(cat image-id) -``` diff --git a/sdk-platform-java/hermetic_build/README.md b/sdk-platform-java/hermetic_build/README.md deleted file mode 100644 index e15b7dd22f4d..000000000000 --- a/sdk-platform-java/hermetic_build/README.md +++ /dev/null @@ -1,412 +0,0 @@ -> [!IMPORTANT] -> All scripts/examples assume you are inside the repository root folder. - -# Generate a repository containing GAPIC Client Libraries - -Running the docker image built from `hermetic_build/library_generation` -directory, you can generate a repository containing GAPIC client libraries (a -monorepo, for example, google-cloud-java) from a configuration file. - -## Environment - -- OS: Linux -- Docker - -## Prerequisites - -### Generation configuration - -A generation configuration file is required to generate GAPIC Client Libraries. - -Please refer to [Generation configuration](#configuration-to-generate-a-repository) -for more information. - -### versions.txt - -In order to generate a version for each library, a `versions.txt` has to exist -in `repository-path`. -Please refer to [Repository path](#repository-path-repository-path-optional) -for more information. - -## Parameters to generate a repository using the docker image - -### Generation configuration yaml (`generation-config-path`), optional - -An absolute or relative path to a configuration file containing parameters to -generate the repository. -Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) -for more information. - -The default value is `$(pwd)/generation_config.yaml`, i.e., `generation_config.yaml` -in the current working directory. - -This will raise `FileNotFoundError` if the specified generation config does not -exist or, in case `generation-config-path` is not specified, the default -generation config does not exist. - -### Repository path (`repository-path`), optional - -The path to where the generated repository goes. - -The default value is the current working directory when running the script. -For example, `cd google-cloud-java && python /path/to/entry_point.py ...` without -specifying the `--repository_path` option will modify the `google-cloud-java` -repository the user `cd`'d into. - -Note that `versions.txt` has to exist in `repository_path` in order to generate -the right version for each library. -Please refer go/java-client-releasing#versionstxt-manifest for more info -of versions.txt. - -### A list of library names (`library-names`), optional - -A list of library names that will be generated, separated by comma. -The library name of a library is the value of `library_name` or `api_shortname`, -if `library_name` is not specified, in the generation configuration. - -If not specified, all libraries in the generation -configuration will be generated. - -### Api definitions path (`api-definitions-path`), optional - -The path to where the api definition (proto, service yaml) resides. - -The default value is the current working directory when running the script. - -**Note that you need not only the protos defined the service, but also the -transitive dependencies of those protos.** -Any missing dependencies will cause `File not found` error. - -For example, if your service is defined in `example_service.proto` and it -imports `google/api/annotations.proto`, you need the `annotations.proto` resides -in a folder that has the exact structure of the import statement (`google/api` -in this case), and set `api_definitions_path` to the path contains the root -folder (`google` in this case). - -## Output - -### GAPIC libraries - -For each module (e.g. `google-cloud-java/java-asset`), the following -files/folders will be created/modified: - -| Name | Notes | -|:------------------------------------|:-------------------------------------------------------------------------| -| google-*/ | Source code generated by gapic-generator-java | -| google-*/pom.xml | Only be generated if it does not exist | -| grpc-*/ | Source code generated by grpc generator, one per each version | -| grpc-*/pom.xml | Only be generated if it does not exist | -| proto-*/ | Source code generated by Protobuf default compiler, one per each version | -| proto-*/pom.xml | Only be generated if it does not exist | -| samples/snippets/generated/ | Only be generated if `include_samples` is set to true | -| google-*-bom/pom.xml | Library BOM, only be generated if it does not exist | -| pom.xml | Library parent BOM, only be generated if it does not exist | -| .repo-metadata.json | Always generated from inputs | -| .OwlBot-hermetic.yaml | Only be generated from a template if it does not exist | -| owlbot.py | Only be generated from a template if it does not exist | -| README.md | Always generated from inputs | -| gapic-libraries-bom/pom.xml | Always generated from inputs | -| pom.xml (repo root dir) | Always generated from inputs | -| versions.txt | New entries will be added if they don’t exist | - -## Configuration to generate a repository - -There are three levels of parameters in the configuration: repository level, -library level and GAPIC level. - -### Repository level parameters - -The repository level parameters define the version of API definition (proto) -and tools. -They are shared by library level parameters. - -| Name | Required | Notes | -|:------------------------|:--------:|:------------------------------------------| -| gapic_generator_version | No | set through env variable if not specified | -| googleapis_commitish | Yes | | -| libraries_bom_version | No | empty string if not specified | - -### Library level parameters - -The library level parameters define how to generate a (multi-versions) GAPIC -library. -They are shared by all GAPICs of a library. - -| Name | Required | Notes | -|:----------------------|:--------:|:------------------------------------------------------------------------------------------------------------------------------------------| -| api_shortname | Yes | | -| api_description | Yes | | -| name_pretty | Yes | | -| product_docs | Yes | | -| library_type | No | `GAPIC_AUTO` if not specified | -| release_level | No | `preview` if not specified | -| api_id | No | `{api_shortname}.googleapis.com` if not specified | -| api_reference | No | | -| codeowner_team | No | | -| client_documentation | No | | -| distribution_name | No | `{group_id}:google-{cloud_prefix}{library_name}` if not specified | -| excluded_poms | No | | -| excluded_dependencies | No | | -| googleapis_commitish | No | use repository level `googleapis_commitish` if not specified. | -| group_id | No | `com.google.cloud` if not specified | -| issue_tracker | No | | -| library_name | No | `api_shortname` is not specified. This value should be unique among all libraries. | -| rest_documentation | No | | -| rpc_documentation | No | | -| cloud_api | No | `true` if not specified | -| requires_billing | No | `true` if not specified | -| transport | No | must be one of `grpc`, `rest` or `both`. This value would only be used for generating .repo-metadata.json and relevant sections in README | - - -Note that `cloud_prefix` is `cloud-` if `cloud_api` is `true`; empty otherwise. - -### GAPIC level parameters - -The GAPIC level parameters define how to generate a GAPIC library. - -| Name | Required | Notes | -|:-----------|:--------:|:------------------------------------------| -| proto_path | Yes | versioned proto_path starts with `google` | - -### An example of generation configuration - -```yaml -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." - release_level: "stable" - library_name: "apigee-connect" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 - - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 -``` - -# Run the library generation image - -1. Download the API definitions to a local directory, e.g., from [googleapis](https://github.com/googleapis/googleapis). - -2. Run the docker image. - ```shell - # Assume you want to generate the library in the current working directory - # and the generation configuration is in the same directory. - docker run \ - --rm \ - --quiet \ - -u "$(id -u):$(id -g)" \ - -v "$(pwd):/workspace" \ - -v /path/to/api_definition:/workspace/apis \ - gcr.io/cloud-devrel-public-resources/java-library-generation:image-tag - ``` - - * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating - yourself. - This avoids folder ownership changes since it runs as root by default. - * `-v "$(pwd):/workspace"` maps the host machine's current working directory - to the /workspace folder. - The image is configured to perform changes in this directory. - * `-v /path/to/api_definition:/workspace/apis` maps the host machine's API - definitions folder to `/workspace/apis` folder. - -3. An advanced example: - ```shell - docker run \ - --rm \ - --quiet \ - -u "$(id -u):$(id -g)" \ - -v "$(pwd):/workspace" \ - -v /path/to/api_definition:/workspace/apis \ - gcr.io/cloud-devrel-public-resources/java-library-generation:image-tag \ - --generation-config-path=/workspace/generation_config_file \ - --library-names=apigee-connect,asset \ - --repository-path=/workspace \ - --api-definitions-path=/workspace/apis - ``` - - * `--generation-config-path=/workspace/generation_config_file` set the - generation configuration to `/workspace/generation_config_file`. - * `--api-definitions-path=/workspace/apis` set the API definition path to - `/workspace/apis`. - -To debug the image, please refer to [development guide](DEVELOPMENT.md#debug-the-library-generation-container) -for more info. - -## An example to generate a repository using the docker image - -If you run the docker image with the example [configuration](#an-example-of-generation-configuration) -shown above, the repository structure is: -``` -$repository_path -|_gapic-libraries-bom -| |_pom.xml -|_java-apigee-connect -| |_google-cloud-apigee-connect -| | |_src -| | |_pom.xml -| |_google-cloud-apigee-connect-bom -| | |_pom.xml -| |_grpc-google-cloud-apigee-connect-v1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-apigee-connect-v1 -| | |_src -| | |_pom.xml -| |_samples -| | |_snippets -| | | |_generated -| |_.OwlBot-hermetic.yaml -| |_.repo-metadata.json -| |_owlbot.py -| |_pom.xml -| |_README.md -|_java-asset -| |_google-cloud-asset -| | |_src -| | |_pom.xml -| |_google-cloud-asset-bom -| | |_pom.xml -| |_grpc-google-cloud-asset-v1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p1beta1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p2beta1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p5beta1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p7beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p1beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p2beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p5beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p7beta1 -| | |_src -| | |_pom.xml -| |_samples -| | |_snippets -| | | |_generated -| |_.OwlBot-hermetic.yaml -| |_.repo-metadata.json -| |_owlbot.py -| |_pom.xml -| |_README.md -|_pom.xml -|_versions.txt -``` - -# Generate release notes from API definition changes - -The script, `hermetic_build/release_note_generation/cli/generate_release_note.py` -allows you to generate a file containing release notes from API definition -changes in [googleapis](https://github.com/googleapis/googleapis) GitHub -repository. - -## Environment - -- OS: Linux -- Python (3.12.0 or above) - -## Parameters to generate a release note - -### Baseline generation configuration path (`baseline-generation-config-path`) - -Absolute or relative path to a generation configuration. -Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) -for more information. - -Note that the `googleapis_commitish` in this configuration is used to retrieve -the first commit, exclusively, to generate the release notes. - -### Current generation configuration path (`current-generation-config-path`) - -Absolute or relative path to a generation configuration. -The release notes will be generated from commits that are related to the -libraries specified in this configuration. -Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) -for more information. - -Note that the `googleapis_commitish` entry in this configuration is used to -retrieve the last commit, inclusively, to generate the release notes. - -### Repository path (`repository-path`), optional - -The path to which the file, `pr_description.txt` containing the release notes -will be sent. -If not specified, the file will be generated to the current working directory. - -## Generate a release notes file in a local environment - -1. Install python (>= 3.12.0). -It is recommended to create a python virtual environment through the -[official guide](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments). - -2. Run the following commands to install python packages - ```shell - cd /path/to/sdk-platform-java - pip install --require-hashes -r hermetic_build/common/requirements.txt - pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt - pip install hermetic_build/release_note_generation - ``` -3. Run the following commands to generate a release note - ```shell - cd /path/to/sdk-platform-java - python hermetic_build/release_note_generation/cli/generate_release_note.py generate \ - --baseline-generation-config-path=/path/to/baseline_generation_config \ - --current-generation-config-path=/path/to/current_generation_config \ - --repository-path=/path/to/send/release_note - ``` - -# OwlBot Java Postprocessor - -We have transferred the -[implementation](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) -of Java Owlbot Postprocessor into `sdk-platform-java/hermetic_build/library_generation`. - -## Reflecting changes in synthtool/docker/owlbot/java into this repository -The transfer was not a verbatim copy, it rather had modifications: - * `format-source.sh` was replaced by a call to `mvn fmt:format` - * `entrypoint.sh` was modified to have input arguments and slightly modified - the way the helper scripts are called - * Other helper scripts were modified to have input arguments. - * `fix_poms.py` modified the way the monorepo is detected - -All these modifications imply that whenever we want to reflect a change from the -original owlbot in synthtool we may be better off modifying the affected source -files one by one. The mapping is from -[`synthtool/docker/owlbot/java`](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) -to -[`sdk-platform-java/library_generation/owlbot`](https://github.com/googleapis/sdk-platform-java/tree/move-java-owlbot/library_generation/owlbot) diff --git a/sdk-platform-java/hermetic_build/common/__init__.py b/sdk-platform-java/hermetic_build/common/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/common/cli/get_changed_libraries.py b/sdk-platform-java/hermetic_build/common/cli/get_changed_libraries.py deleted file mode 100644 index 4f2bfe45c65e..000000000000 --- a/sdk-platform-java/hermetic_build/common/cli/get_changed_libraries.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Setup logging -import logging -import sys - -logging.basicConfig(level=logging.DEBUG, stream=sys.stderr) - -import os -import click - -from common.model.config_change import ConfigChange -from common.model.generation_config import GenerationConfig -from common.utils.generation_config_comparator import compare_config - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--baseline-generation-config-path", - required=True, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - This config file is used for computing changed library list. - """, -) -@click.option( - "--current-generation-config-path", - required=True, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml that contains the - metadata about library generation. - """, -) -@click.option( - "--force-regenerate-all", - required=False, - type=bool, - help=""" - Force regenerate all libraries. - """, -) -def create( - baseline_generation_config_path: str, - current_generation_config_path: str, - force_regenerate_all: bool, -) -> None: - """ - Compares baseline generation config with current generation config and - generates changed library names (a comma separated string) based on current - generation config. - """ - baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) - if not os.path.isfile(baseline_generation_config_path): - raise FileNotFoundError( - f"{baseline_generation_config_path} does not exist. " - "A valid generation config has to be passed in as " - "baseline-generation-config-path." - ) - current_generation_config_path = os.path.abspath(current_generation_config_path) - if not os.path.isfile(current_generation_config_path): - raise FileNotFoundError( - f"{current_generation_config_path} does not exist. " - "A valid generation config has to be passed in as " - "current-generation-config-path." - ) - config_change = compare_config( - baseline_config=GenerationConfig.from_yaml(baseline_generation_config_path), - current_config=GenerationConfig.from_yaml(current_generation_config_path), - force_regenerate_all=force_regenerate_all, - ) - click.echo(",".join(config_change.get_changed_libraries())) - - -if __name__ == "__main__": - main() diff --git a/sdk-platform-java/hermetic_build/common/model/config_change.py b/sdk-platform-java/hermetic_build/common/model/config_change.py deleted file mode 100644 index 401d943ff62d..000000000000 --- a/sdk-platform-java/hermetic_build/common/model/config_change.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import tempfile -from enum import Enum -from typing import Optional -from git import Commit, Repo -from common.model.gapic_inputs import parse_build_str -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig -from common.utils.proto_path_utils import find_versioned_proto_path - -INSERTIONS = "insertions" -LINES = "lines" - - -class ChangeType(Enum): - GOOGLEAPIS_COMMIT = 1 - REPO_LEVEL_CHANGE = 2 - LIBRARIES_ADDITION = 3 - # As of Mar. 2024, we decide not to produce this type of change because we - # still need to manually remove the libray. - # LIBRARIES_REMOVAL = 4 - LIBRARY_LEVEL_CHANGE = 5 - GAPIC_ADDITION = 6 - # As of Mar. 2024, we decide not to produce this type of change because we - # still need to manually remove the libray. - # GAPIC_REMOVAL = 7 - - -class HashLibrary: - """ - Data class to group a LibraryConfig object and its hash value together. - """ - - def __init__(self, hash_value: int, library: LibraryConfig): - self.hash_value = hash_value - self.library = library - - -class LibraryChange: - def __init__(self, changed_param: str, current_value: str, library_name: str = ""): - self.changed_param = changed_param - self.current_value = current_value - self.library_name = library_name - - -class QualifiedCommit: - def __init__(self, commit: Commit, libraries: set[str]): - self.commit = commit - self.libraries = libraries - - -class ConfigChange: - - def __init__( - self, - change_to_libraries: dict[ChangeType, list[LibraryChange]], - baseline_config: GenerationConfig, - current_config: GenerationConfig, - ): - self.change_to_libraries = change_to_libraries - self.baseline_config = baseline_config - self.current_config = current_config - - def get_changed_libraries(self) -> list[str]: - """ - Returns a unique, sorted list of library name of changed libraries. - - :return: library names of change libraries. - """ - if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries: - return [ - library.get_library_name() for library in self.current_config.libraries - ] - library_names = set() - for change_type, library_changes in self.change_to_libraries.items(): - if change_type == ChangeType.GOOGLEAPIS_COMMIT: - library_names.update(self.__get_library_names_from_qualified_commits()) - else: - library_names.update( - [library_change.library_name for library_change in library_changes] - ) - return sorted(list(library_names)) - - def get_qualified_commits( - self, - repo_url: str = "https://github.com/googleapis/googleapis.git", - ) -> list[QualifiedCommit]: - """ - Returns qualified commits from configuration change. - - A qualified commit is a commit that changes at least one file (excluding - BUILD.bazel) within a versioned proto path in the given proto_paths. - :param repo_url: the repository contains the commit history. - :return: QualifiedCommit objects. - """ - with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_dir: - # We include blobs to avoid unexpected missing git objects when computing - # a diff. - # See https://github.com/googleapis/sdk-platform-java/issues/3745 - repo = Repo.clone_from(url=repo_url, to_path=tmp_dir) - commit = repo.commit(self.current_config.googleapis_commitish) - proto_paths = self.current_config.get_proto_path_to_library_name() - qualified_commits = [] - while str(commit.hexsha) != self.baseline_config.googleapis_commitish: - qualified_commit = ConfigChange.__create_qualified_commit( - proto_paths=proto_paths, commit=commit - ) - if qualified_commit is not None: - qualified_commits.append(qualified_commit) - commit_parents = commit.parents - if len(commit_parents) == 0: - break - commit = commit_parents[0] - repo.close() - return qualified_commits - - def __get_library_names_from_qualified_commits(self) -> list[str]: - qualified_commits = self.get_qualified_commits() - library_names = [] - for qualified_commit in qualified_commits: - library_names.extend(qualified_commit.libraries) - return library_names - - @staticmethod - def __create_qualified_commit( - proto_paths: dict[str, str], commit: Commit - ) -> Optional[QualifiedCommit]: - """ - Returns a qualified commit from the given Commit object; otherwise None. - - :param proto_paths: a mapping from versioned proto_path to library_name - :param commit: a GitHub commit object. - :return: qualified commits. - """ - libraries = set() - for file_path, changes in commit.stats.files.items(): - versioned_proto_path = find_versioned_proto_path(file_path) - if versioned_proto_path in proto_paths: - if ( - file_path.endswith("BUILD.bazel") - # Qualify a commit if the commit only added BUILD.bazel - # because it's very unlikely that a commit added BUILD.bazel - # without adding proto files. Therefore, the commit is - # qualified duo to the proto change eventually. - and (not ConfigChange.__is_added(changes)) - and ( - not ConfigChange.__is_qualified_build_change( - commit=commit, build_file_path=file_path - ) - ) - ): - continue - # Even though a commit usually only changes one - # library, we don't want to miss generating a - # library because the commit may change multiple - # libraries. - libraries.add(proto_paths[versioned_proto_path]) - if len(libraries) == 0: - return None - return QualifiedCommit(commit=commit, libraries=libraries) - - @staticmethod - def __is_added(changes: dict[str, int]) -> bool: - return changes[INSERTIONS] == changes[LINES] - - @staticmethod - def __is_qualified_build_change(commit: Commit, build_file_path: str) -> bool: - """ - Checks if the given commit containing a BUILD.bazel change is a - qualified commit. - - The commit is a qualified commit if the - :class:`library_generation.model.gapic_inputs.GapicInputs` objects - parsed from the commit and its parent are different, since there are - changes in fields that used in library generation. - - :param commit: a GitHub commit object. - :param build_file_path: the path of the BUILD.bazel - :return: True if the commit is a qualified commit; False otherwise. - """ - versioned_proto_path = find_versioned_proto_path(build_file_path) - build = str((commit.tree / build_file_path).data_stream.read()) - parent_commit = commit.parents[0] - parent_build = str((parent_commit.tree / build_file_path).data_stream.read()) - inputs = parse_build_str(build, versioned_proto_path) - parent_inputs = parse_build_str(parent_build, versioned_proto_path) - return inputs != parent_inputs diff --git a/sdk-platform-java/hermetic_build/common/model/gapic_config.py b/sdk-platform-java/hermetic_build/common/model/gapic_config.py deleted file mode 100644 index 3bbba394547a..000000000000 --- a/sdk-platform-java/hermetic_build/common/model/gapic_config.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re -from typing import Optional - -ALPHA_VERSION = "alpha" -BETA_VERSION = "beta" - - -class GapicConfig: - """ - Class that represents a GAPICs single entry, inside a `LibraryConfig` in - a generation_config.yaml - """ - - def __init__(self, proto_path: str): - self.proto_path = proto_path - self.version = self.__parse_version() - - def is_stable(self): - return ( - self.version - and (ALPHA_VERSION not in self.version) - and (BETA_VERSION not in self.version) - ) - - def get_version(self): - return self.version - - def __parse_version(self) -> Optional[str]: - version_regex = re.compile(r"^v[1-9]+(p[1-9]+)*(alpha|beta)?.*") - for directory in self.proto_path.split("/"): - if version_regex.search(directory): - return directory - return None - - def __eq__(self, other) -> bool: - if not isinstance(other, GapicConfig): - return False - return self.proto_path == other.proto_path - - def __lt__(self, other) -> bool: - if not isinstance(other, GapicConfig): - raise ValueError( - f"Type {type(other)} can't be comparable " f"with GapicConfig." - ) - - self_version = self.get_version() - other_version = other.get_version() - self_dirs = self.proto_path.split("/") - other_dirs = other.proto_path.split("/") - # Case 1: if both of the configs don't have a version in proto_path, - # the one with lower depth is smaller. - if (not self_version) and (not other_version): - return len(self_dirs) < len(other_dirs) - # Case 2: if only one config has a version in proto_path, it is smaller - # than the other one. - if self_version and (not other_version): - return True - if (not self_version) and other_version: - return False - # Two configs both have a version in proto_path. - self_stable = self.is_stable() - other_stable = other.is_stable() - # Case 3, if only config has a stable version in proto_path, it is - # smaller than the other one. - if self_stable and (not other_stable): - return True - if (not self_stable) and other_stable: - return False - # Case 4, if two configs have a non-stable version in proto_path, - # the one with higher version is smaller. - # Note that using string comparison may lead unexpected result, - # e.g., v1beta10 is smaller than v1beta2. - # In reality, however, there's unlikely that a library has >10 beta - # versions. - if (not self_stable) and (not other_stable): - return self_version > other_version - # Two configs both have a stable version in proto_path. - # Case 5, if two configs have different depth in proto_path, the one - # with lower depth is smaller. - if len(self_dirs) != len(other_dirs): - return len(self_dirs) < len(other_dirs) - # Case 6, the config with higher stable version is smaller. - self_num = int(self_version[1:]) - other_num = int(other_version[1:]) - return self_num > other_num diff --git a/sdk-platform-java/hermetic_build/common/model/gapic_inputs.py b/sdk-platform-java/hermetic_build/common/model/gapic_inputs.py deleted file mode 100644 index 5c07777965f0..000000000000 --- a/sdk-platform-java/hermetic_build/common/model/gapic_inputs.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -import re - -proto_library_pattern = r""" -proto_library_with_info\( -(.*?) -\) -""" -gapic_pattern = r""" -java_gapic_library\( -(.*?) -\) -""" -assembly_pattern = r""" -java_gapic_assembly_gradle_pkg\( -(.*?) -\) -""" -# match a line which the first character is "#". -comment_pattern = r"^\s*\#+" -pattern_to_proto = { - r"//google/cloud:common_resources_proto": "google/cloud/common_resources.proto", - r"//google/cloud/location:location_proto": "google/cloud/location/locations.proto", - r"//google/iam/v1:iam_policy_proto": "google/iam/v1/iam_policy.proto", -} -transport_pattern = r"transport = \"(.*?)\"" -rest_pattern = r"rest_numeric_enums = True" -gapic_yaml_pattern = r"gapic_yaml = \"(.*?)\"" -service_config_pattern = r"grpc_service_config = \"(.*?)\"" -service_yaml_pattern = r"service_yaml = \"(.*?)\"" -include_samples_pattern = r"include_samples = True" - - -class GapicInputs: - """ - A data class containing inputs to invoke generate_library.sh to generate - a GAPIC library. - """ - - def __init__( - self, - proto_only="true", - additional_protos="google/cloud/common_resources.proto", - transport="grpc", - rest_numeric_enum="", - gapic_yaml="", - service_config="", - service_yaml="", - include_samples="true", - ): - self.proto_only = proto_only - self.additional_protos = additional_protos - self.transport = transport - self.rest_numeric_enum = rest_numeric_enum - self.gapic_yaml = gapic_yaml - self.service_config = service_config - self.service_yaml = service_yaml - self.include_samples = include_samples - - def __eq__(self, other): - if not isinstance(other, GapicInputs): - return False - return ( - self.proto_only == other.proto_only - and self.additional_protos == other.additional_protos - and self.transport == other.transport - and self.rest_numeric_enum == other.rest_numeric_enum - and self.gapic_yaml == other.gapic_yaml - and self.service_config == other.service_config - and self.service_yaml == other.service_yaml - and self.include_samples == other.include_samples - ) - - -def parse( - build_path: Path, versioned_path: str, build_file_name: str = "BUILD.bazel" -) -> GapicInputs: - """ - Utility function to parse inputs of generate_library.sh from BUILD.bazel. - :param build_path: the file path of BUILD.bazel - :param versioned_path: a versioned path in googleapis repository, e.g., - google/cloud/asset/v1. - :param build_file_name: the name of the build file. - :return: an GapicInputs object. - """ - with open(f"{build_path}/{build_file_name}") as build: - content = build.read() - return parse_build_str(build_str=content, versioned_path=versioned_path) - - -def parse_build_str(build_str: str, versioned_path: str) -> GapicInputs: - proto_library_target = re.compile( - proto_library_pattern, re.DOTALL | re.VERBOSE - ).findall(build_str) - additional_protos = "" - if len(proto_library_target) > 0: - additional_protos = __parse_additional_protos(proto_library_target[0]) - gapic_target = re.compile(gapic_pattern, re.DOTALL | re.VERBOSE).findall(build_str) - assembly_target = re.compile(assembly_pattern, re.DOTALL | re.VERBOSE).findall( - build_str - ) - include_samples = "false" - if len(assembly_target) > 0: - include_samples = __parse_include_samples(assembly_target[0]) - if len(gapic_target) == 0: - return GapicInputs( - additional_protos=additional_protos, include_samples=include_samples - ) - - transport = __parse_transport(gapic_target[0]) - rest_numeric_enum = __parse_rest_numeric_enums(gapic_target[0]) - gapic_yaml = __parse_gapic_yaml(gapic_target[0], versioned_path) - service_config = __parse_service_config(gapic_target[0], versioned_path) - service_yaml = __parse_service_yaml(gapic_target[0], versioned_path) - - return GapicInputs( - proto_only="false", - additional_protos=additional_protos, - transport=transport, - rest_numeric_enum=rest_numeric_enum, - gapic_yaml=gapic_yaml, - service_config=service_config, - service_yaml=service_yaml, - include_samples=include_samples, - ) - - -def __parse_additional_protos(proto_library_target: str) -> str: - res = [" "] - lines = proto_library_target.split("\n") - for line in lines: - if len(re.findall(comment_pattern, line)) != 0: - # skip a line which the first charactor is "#" since it's - # a comment. - continue - for pattern in pattern_to_proto: - if len(re.findall(pattern, line)) == 0: - continue - res.append(pattern_to_proto[pattern]) - return " ".join(res) - - -def __parse_transport(gapic_target: str) -> str: - transport = re.findall(transport_pattern, gapic_target) - return transport[0] if len(transport) != 0 else "grpc" - - -def __parse_rest_numeric_enums(gapic_target: str) -> str: - rest_numeric_enums = re.findall(rest_pattern, gapic_target) - return "true" if len(rest_numeric_enums) != 0 else "false" - - -def __parse_gapic_yaml(gapic_target: str, versioned_path: str) -> str: - gapic_yaml = re.findall(gapic_yaml_pattern, gapic_target) - return f"{versioned_path}/{gapic_yaml[0]}" if len(gapic_yaml) != 0 else "" - - -def __parse_service_config(gapic_target: str, versioned_path: str) -> str: - service_config = re.findall(service_config_pattern, gapic_target) - return ( - f"{versioned_path}/{service_config[0]}".replace(":", "") - if len(service_config) != 0 - else "" - ) - - -def __parse_service_yaml(gapic_target: str, versioned_path: str) -> str: - service_yaml = re.findall(service_yaml_pattern, gapic_target) - if len(service_yaml) == 0: - return "" - res = str(service_yaml[0]) - if res.startswith("//"): - # special case if the service config starts with "//", is a Bazel - # target with an absolute path. - return res.replace("//", "").replace(":", "/") - return f"{versioned_path}/{res}" - - -def __parse_include_samples(assembly_target: str) -> str: - include_samples = re.findall(include_samples_pattern, assembly_target) - return "true" if len(include_samples) != 0 else "false" diff --git a/sdk-platform-java/hermetic_build/common/model/generation_config.py b/sdk-platform-java/hermetic_build/common/model/generation_config.py deleted file mode 100644 index fabbde41ccae..000000000000 --- a/sdk-platform-java/hermetic_build/common/model/generation_config.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -import yaml -from typing import Optional -from common.model.library_config import LibraryConfig -from common.model.gapic_config import GapicConfig - -REPO_LEVEL_PARAMETER = "Repo level parameter" -LIBRARY_LEVEL_PARAMETER = "Library level parameter" -GAPIC_LEVEL_PARAMETER = "GAPIC level parameter" -COMMON_PROTOS_LIBRARY_NAME = "common-protos" -GAPIC_GENERATOR_VERSION = "gapic_generator_version" -LIBRARIES_BOM_VERSION = "libraries_bom_version" -GENERATOR_VERSION_ENV_KEY = "GENERATOR_VERSION" - - -class GenerationConfig: - """ - Class that represents the root of a generation_config.yaml - """ - - def __init__( - self, - googleapis_commitish: str, - libraries: list[LibraryConfig], - gapic_generator_version: Optional[str] = None, - libraries_bom_version: Optional[str] = None, - ): - self.googleapis_commitish = googleapis_commitish - self.libraries_bom_version = ( - libraries_bom_version if libraries_bom_version else "" - ) - self.gapic_generator_version = GenerationConfig.__set_generator_version( - gapic_generator_version - ) - self.libraries = libraries - # explicit set to None so that we can compute the - # value in getter. - self.__contains_common_protos = None - self.__validate() - - def get_proto_path_to_library_name(self) -> dict[str, str]: - """ - Get versioned proto_path to library_name mapping from configuration. - - :return: versioned proto_path to library_name mapping - """ - paths = {} - for library in self.libraries: - for gapic_config in library.gapic_configs: - paths[gapic_config.proto_path] = library.get_library_name() - return paths - - def is_monorepo(self) -> bool: - return len(self.libraries) > 1 - - def contains_common_protos(self) -> bool: - if self.__contains_common_protos is None: - self.__contains_common_protos = False - for library in self.libraries: - if library.get_library_name() == COMMON_PROTOS_LIBRARY_NAME: - self.__contains_common_protos = True - break - return self.__contains_common_protos - - @staticmethod - def __set_generator_version(gapic_generator_version: Optional[str]) -> str: - if gapic_generator_version is not None: - return gapic_generator_version - # if the generator version is not set through generation config, - # get it from environment variable. - gapic_generator_version = os.getenv(GENERATOR_VERSION_ENV_KEY) - if not gapic_generator_version: - raise ValueError( - f"Environment variable {GENERATOR_VERSION_ENV_KEY}" - f" is not set when the generator version is not" - f" specified in the generation config." - ) - return gapic_generator_version - - def __validate(self) -> None: - seen_library_names = dict() - for library in self.libraries: - library_name = library.get_library_name() - if library_name in seen_library_names: - raise ValueError( - f"Both {library.name_pretty} and " - f"{seen_library_names.get(library_name)} have the same " - f"library name: {library_name}, please update one of the " - f"library to have a different library name." - ) - seen_library_names[library_name] = library.name_pretty - - @staticmethod - def from_yaml(path_to_yaml: str) -> "GenerationConfig": - """ - Parses a yaml located in path_to_yaml. - :param path_to_yaml: the path to the configuration file - :return the parsed configuration represented by the "model" classes - """ - with open(path_to_yaml, "r") as file_stream: - config = yaml.safe_load(file_stream) - - libraries = _required(config, "libraries", REPO_LEVEL_PARAMETER) - if not libraries: - raise ValueError(f"Library is None in {path_to_yaml}.") - - parsed_libraries = list() - for library in libraries: - gapics = _required(library, "GAPICs") - - parsed_gapics = list() - if not gapics: - raise ValueError(f"GAPICs is None in {library}.") - for gapic in gapics: - proto_path = _required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER) - new_gapic = GapicConfig(proto_path) - parsed_gapics.append(new_gapic) - - new_library = LibraryConfig( - api_shortname=_required(library, "api_shortname"), - api_description=_required(library, "api_description"), - name_pretty=_required(library, "name_pretty"), - product_documentation=_required(library, "product_documentation"), - gapic_configs=parsed_gapics, - library_type=_optional(library, "library_type", "GAPIC_AUTO"), - release_level=_optional(library, "release_level", "preview"), - api_id=_optional(library, "api_id", None), - api_reference=_optional(library, "api_reference", None), - codeowner_team=_optional(library, "codeowner_team", None), - excluded_poms=_optional(library, "excluded_poms", None), - excluded_dependencies=_optional(library, "excluded_dependencies", None), - client_documentation=_optional(library, "client_documentation", None), - distribution_name=_optional(library, "distribution_name", None), - googleapis_commitish=_optional(library, "googleapis_commitish", None), - group_id=_optional(library, "group_id", "com.google.cloud"), - issue_tracker=_optional(library, "issue_tracker", None), - library_name=_optional(library, "library_name", None), - rest_documentation=_optional(library, "rest_documentation", None), - rpc_documentation=_optional(library, "rpc_documentation", None), - cloud_api=_optional(library, "cloud_api", True), - requires_billing=_optional(library, "requires_billing", True), - extra_versioned_modules=_optional( - library, "extra_versioned_modules", None - ), - recommended_package=_optional(library, "recommended_package", None), - min_java_version=_optional(library, "min_java_version", None), - transport=_optional(library, "transport", None), - ) - parsed_libraries.append(new_library) - - parsed_config = GenerationConfig( - googleapis_commitish=_required( - config, "googleapis_commitish", REPO_LEVEL_PARAMETER - ), - gapic_generator_version=_optional(config, GAPIC_GENERATOR_VERSION, None), - libraries_bom_version=_optional(config, LIBRARIES_BOM_VERSION, None), - libraries=parsed_libraries, - ) - - return parsed_config - - -def _required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER): - if key not in config: - message = ( - f"{level}, {key}, is not found in {config} in yaml." - if level != REPO_LEVEL_PARAMETER - else f"{level}, {key}, is not found in yaml." - ) - raise ValueError(message) - return config[key] - - -def _optional(config: dict, key: str, default: any): - if key not in config: - return default - return config[key] diff --git a/sdk-platform-java/hermetic_build/common/model/library_config.py b/sdk-platform-java/hermetic_build/common/model/library_config.py deleted file mode 100644 index dde5c24feb65..000000000000 --- a/sdk-platform-java/hermetic_build/common/model/library_config.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from hashlib import sha256 -from typing import Optional -from common.model.gapic_config import GapicConfig -from common.model.gapic_inputs import GapicInputs - -MAVEN_COORDINATE_SEPARATOR = ":" - - -class LibraryConfig: - """ - Class that represents a library in a generation_config.yaml file - """ - - def __init__( - self, - api_shortname: str, - api_description: str, - name_pretty: str, - product_documentation: str, - gapic_configs: list[GapicConfig], - library_type: Optional[str] = None, - release_level: Optional[str] = None, - api_id: Optional[str] = None, - api_reference: Optional[str] = None, - codeowner_team: Optional[str] = None, - client_documentation: Optional[str] = None, - distribution_name: Optional[str] = None, - excluded_dependencies: Optional[str] = None, - excluded_poms: Optional[str] = None, - googleapis_commitish: Optional[str] = None, - group_id: Optional[str] = "com.google.cloud", - issue_tracker: Optional[str] = None, - library_name: Optional[str] = None, - rest_documentation: Optional[str] = None, - rpc_documentation: Optional[str] = None, - cloud_api: Optional[bool] = True, - requires_billing: Optional[bool] = True, - extra_versioned_modules: Optional[str] = None, - recommended_package: Optional[str] = None, - min_java_version: Optional[int] = None, - transport: Optional[str] = None, - ): - self.api_shortname = api_shortname - self.api_description = api_description - self.name_pretty = name_pretty - self.product_documentation = product_documentation - self.gapic_configs = gapic_configs - self.library_type = library_type if library_type else "GAPIC_AUTO" - self.release_level = release_level if release_level else "preview" - self.api_id = api_id - self.api_reference = api_reference - self.codeowner_team = codeowner_team - self.excluded_dependencies = excluded_dependencies - self.excluded_poms = excluded_poms - self.client_documentation = client_documentation - self.googleapis_commitish = googleapis_commitish - self.group_id = group_id - self.issue_tracker = issue_tracker - self.library_name = library_name - self.rest_documentation = rest_documentation - self.rpc_documentation = rpc_documentation - self.cloud_api = cloud_api - self.requires_billing = requires_billing - self.extra_versioned_modules = extra_versioned_modules - self.recommended_package = recommended_package - self.min_java_version = min_java_version - self.distribution_name = self.__get_distribution_name(distribution_name) - self.transport = self.__validate_transport(transport) - - def set_gapic_configs(self, gapic_configs: list[GapicConfig]) -> None: - """ - Sets the gapic_configs for the library. - - Args: - gapic_configs: The new list of GapicConfig objects. - """ - self.gapic_configs = gapic_configs - - def get_library_name(self) -> str: - """ - Return the library name of a given LibraryConfig object - :return: the library name - """ - return self.library_name if self.library_name else self.api_shortname - - def get_sorted_gapic_configs(self) -> list[GapicConfig]: - return sorted(self.gapic_configs) - - def get_maven_coordinate(self) -> str: - """ - Returns the Maven coordinate (group_id:artifact_id) of the library - """ - return self.distribution_name - - def get_artifact_id(self) -> str: - """ - Returns the artifact ID of the library - """ - return self.get_maven_coordinate().split(MAVEN_COORDINATE_SEPARATOR)[-1] - - def get_transport(self, gapic_inputs: GapicInputs) -> str: - """ - Returns the transport of the library. If directly set in library config, return it. - Otherwise, return the transport inferred from gapic_inputs. This value is only - used for postprocessing - the generation still infers the transport from BUILD - files. - """ - return self.transport if self.transport is not None else gapic_inputs.transport - - def __get_distribution_name(self, distribution_name: Optional[str]) -> str: - LibraryConfig.__check_distribution_name(distribution_name) - if distribution_name: - return distribution_name - cloud_prefix = "cloud-" if self.cloud_api else "" - library_name = self.get_library_name() - return f"{self.group_id}:google-{cloud_prefix}{library_name}" - - def __validate_transport(self, transport: str): - if transport not in [None, "grpc", "rest", "both"]: - raise ValueError( - "allowed values for library.transport: grpc, rest, both and None" - ) - return transport - - @staticmethod - def __check_distribution_name(distribution_name: str) -> None: - if not distribution_name: - return - sections = distribution_name.split(MAVEN_COORDINATE_SEPARATOR) - if len(sections) != 2: - raise ValueError(f"{distribution_name} is not a valid distribution name.") - - def __eq__(self, other): - return ( - self.api_shortname == other.api_shortname - and self.api_description == other.api_description - and self.name_pretty == other.name_pretty - and self.product_documentation == other.product_documentation - and self.gapic_configs == other.gapic_configs - and self.library_type == other.library_type - and self.release_level == other.release_level - and self.api_id == other.api_id - and self.api_reference == other.api_reference - and self.codeowner_team == other.codeowner_team - and self.excluded_dependencies == other.excluded_dependencies - and self.excluded_poms == other.excluded_poms - and self.client_documentation == other.client_documentation - and self.distribution_name == other.distribution_name - and self.googleapis_commitish == other.googleapis_commitish - and self.group_id == other.group_id - and self.issue_tracker == other.issue_tracker - and self.library_name == other.library_name - and self.rest_documentation == other.rest_documentation - and self.rpc_documentation == other.rpc_documentation - and self.cloud_api == other.cloud_api - and self.requires_billing == other.requires_billing - and self.extra_versioned_modules == other.extra_versioned_modules - and self.recommended_package == other.recommended_package - and self.min_java_version == other.min_java_version - and self.transport == other.transport - ) - - def __hash__(self): - m = sha256() - m.update( - str( - [ - self.api_shortname, - self.api_description, - self.name_pretty, - self.product_documentation, - self.library_type, - self.release_level, - self.api_id, - self.api_reference, - self.codeowner_team, - self.excluded_dependencies, - self.excluded_poms, - self.client_documentation, - self.distribution_name, - self.googleapis_commitish, - self.group_id, - self.issue_tracker, - self.library_name, - self.rest_documentation, - self.rpc_documentation, - self.cloud_api, - self.requires_billing, - self.extra_versioned_modules, - self.recommended_package, - self.min_java_version, - self.transport, - ] - + [config.proto_path for config in self.gapic_configs] - ).encode("utf-8") - ) - return int(m.hexdigest(), 16) diff --git a/sdk-platform-java/hermetic_build/common/requirements.in b/sdk-platform-java/hermetic_build/common/requirements.in deleted file mode 100644 index 395bee76c60f..000000000000 --- a/sdk-platform-java/hermetic_build/common/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -black==26.3.1 -GitPython==3.1.43 -parameterized==0.9.0 -PyYAML==6.0.2 \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/requirements.txt b/sdk-platform-java/hermetic_build/common/requirements.txt deleted file mode 100644 index 6e779a031429..000000000000 --- a/sdk-platform-java/hermetic_build/common/requirements.txt +++ /dev/null @@ -1,170 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --generate-hashes hermetic_build/common/requirements.in -# -black==26.3.1 \ - --hash=sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c \ - --hash=sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7 \ - --hash=sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff \ - --hash=sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b \ - --hash=sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07 \ - --hash=sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78 \ - --hash=sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f \ - --hash=sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5 \ - --hash=sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b \ - --hash=sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e \ - --hash=sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a \ - --hash=sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac \ - --hash=sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a \ - --hash=sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54 \ - --hash=sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2 \ - --hash=sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f \ - --hash=sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1 \ - --hash=sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5 \ - --hash=sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2 \ - --hash=sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f \ - --hash=sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1 \ - --hash=sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c \ - --hash=sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839 \ - --hash=sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983 \ - --hash=sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb \ - --hash=sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56 \ - --hash=sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568 - # via -r requirements.in -click==8.1.7 \ - --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ - --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de - # via black -gitdb==4.0.11 \ - --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ - --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b - # via gitpython -gitpython==3.1.43 \ - --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ - --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff - # via -r requirements.in -mypy-extensions==1.0.0 \ - --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ - --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 - # via black -packaging==24.2 \ - --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ - --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f - # via black -parameterized==0.9.0 \ - --hash=sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b \ - --hash=sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1 - # via -r requirements.in -pathspec==1.0.4 \ - --hash=sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645 \ - --hash=sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723 - # via black -platformdirs==4.3.6 \ - --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ - --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb - # via black -pytokens==0.4.1 \ - --hash=sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1 \ - --hash=sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009 \ - --hash=sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083 \ - --hash=sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1 \ - --hash=sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de \ - --hash=sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2 \ - --hash=sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a \ - --hash=sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1 \ - --hash=sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5 \ - --hash=sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a \ - --hash=sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3 \ - --hash=sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db \ - --hash=sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68 \ - --hash=sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037 \ - --hash=sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321 \ - --hash=sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc \ - --hash=sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7 \ - --hash=sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f \ - --hash=sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918 \ - --hash=sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9 \ - --hash=sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c \ - --hash=sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1 \ - --hash=sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1 \ - --hash=sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3 \ - --hash=sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b \ - --hash=sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb \ - --hash=sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1 \ - --hash=sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a \ - --hash=sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4 \ - --hash=sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa \ - --hash=sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78 \ - --hash=sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe \ - --hash=sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9 \ - --hash=sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d \ - --hash=sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975 \ - --hash=sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440 \ - --hash=sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16 \ - --hash=sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc \ - --hash=sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d \ - --hash=sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6 \ - --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ - --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 - # via black -pyyaml==6.0.2 \ - --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ - --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ - --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ - --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ - --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ - --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ - --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ - --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ - --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ - --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ - --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ - --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ - --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ - --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ - --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ - --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ - --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ - --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ - --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ - --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ - --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ - --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ - --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ - --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ - --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ - --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ - --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ - --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ - --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ - --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ - --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ - --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ - --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ - --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ - --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ - --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ - --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ - --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ - --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ - --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ - --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ - --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ - --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ - --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ - --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ - --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ - --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ - --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ - --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ - --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ - --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ - --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ - --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 - # via -r requirements.in -smmap==5.0.2 \ - --hash=sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5 \ - --hash=sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e - # via gitdb diff --git a/sdk-platform-java/hermetic_build/common/setup.py b/sdk-platform-java/hermetic_build/common/setup.py deleted file mode 100755 index 8c3c663adaf1..000000000000 --- a/sdk-platform-java/hermetic_build/common/setup.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Package information of library_generation python scripts -""" - -from setuptools import setup - -setup( - name="common", - version="0.1", - python_requires=">=3.12", - package_dir={ - "common": ".", - }, -) diff --git a/sdk-platform-java/hermetic_build/common/tests/__init__.py b/sdk-platform-java/hermetic_build/common/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/common/tests/cli/__init__.py b/sdk-platform-java/hermetic_build/common/tests/cli/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/common/tests/cli/config_change_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/cli/config_change_unit_tests.py deleted file mode 100644 index e3bdd753de7b..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/cli/config_change_unit_tests.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from click.testing import CliRunner -import unittest - -from common.cli.get_changed_libraries import create - -script_dir = os.path.dirname(os.path.realpath(__file__)) -test_resource_dir = os.path.join(script_dir, "..", "resources", "cli") - - -class GetChangedLibrariesTest(unittest.TestCase): - def test_entry_point_without_baseline_config_raise_system_exception(self): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke(create) - self.assertEqual(2, result.exit_code) - self.assertEqual(SystemExit, result.exc_info[0]) - - def test_entry_point_without_current_config_raise_system_exception(self): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - create, ["--baseline-generation-config-path=/invalid/path/file"] - ) - self.assertEqual(2, result.exit_code) - self.assertEqual(SystemExit, result.exc_info[0]) - - def test_entry_point_with_invalid_baseline_config_raise_file_exception(self): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - create, - [ - "--baseline-generation-config-path=/invalid/path/file", - "--current-generation-config-path=/invalid/path/file", - ], - ) - self.assertEqual(1, result.exit_code) - self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex(result.exception.args[0], "baseline-generation-config-path") - - def test_entry_point_with_invalid_current_config_raise_file_exception(self): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - create, - [ - f"--baseline-generation-config-path={test_resource_dir}/empty_config.yaml", - "--current-generation-config-path=/invalid/path/file", - ], - ) - self.assertEqual(1, result.exit_code) - self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex(result.exception.args[0], "current-generation-config-path") diff --git a/sdk-platform-java/hermetic_build/common/tests/model/__init__.py b/sdk-platform-java/hermetic_build/common/tests/model/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/common/tests/model/config_change_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/model/config_change_unit_tests.py deleted file mode 100644 index 3cae76626c35..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/model/config_change_unit_tests.py +++ /dev/null @@ -1,328 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from common.model.config_change import ChangeType -from common.model.config_change import ConfigChange -from common.model.config_change import LibraryChange -from common.model.gapic_config import GapicConfig -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig - - -class ConfigChangeTest(unittest.TestCase): - def test_get_changed_libraries_with_repo_level_change_returns_all_libraries_changed( - self, - ): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [], - # add a library level change to verify this type of change has - # no impact on the result. - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="test-library", - ) - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config(), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish="", - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="gke-backup", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/gkebackup/v1") - ], - ), - ConfigChangeTest.__get_a_library_config( - library_name="test-library", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/gkebackup/v1") - ], - ), - ], - ), - ) - self.assertEqual( - ["gke-backup", "test-library"], - config_change.get_changed_libraries(), - ) - - def test_get_changed_libraries_with_library_level_change_returns_list(self): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="a-library", - ), - LibraryChange( - changed_param="test", - current_value="test", - library_name="another-library", - ), - ], - ChangeType.LIBRARIES_ADDITION: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="new-library", - ), - ], - ChangeType.GAPIC_ADDITION: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="library-with-new-version", - ), - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config(), - current_config=ConfigChangeTest.__get_a_gen_config(), - ) - self.assertEqual( - ["a-library", "another-library", "library-with-new-version", "new-library"], - config_change.get_changed_libraries(), - ) - - def test_get_changed_libraries_with_duplicated_library_name_returns_unique_name( - self, - ): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="a-param", - current_value="new_test", - library_name="a-library", - ), - LibraryChange( - changed_param="another-param", - current_value="new_value", - library_name="a-library", - ), - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config(), - current_config=ConfigChangeTest.__get_a_gen_config(), - ) - self.assertEqual( - ["a-library"], - config_change.get_changed_libraries(), - ) - - def test_get_changed_libraries_with_mix_changes_returns_list(self): - baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" - latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" - config_change = ConfigChange( - change_to_libraries={ - ChangeType.GOOGLEAPIS_COMMIT: [], - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="a-library", - ) - ], - ChangeType.LIBRARIES_ADDITION: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="new-library", - ), - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="gke-backup", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/gkebackup/v1") - ], - ), - ], - ), - ) - - self.assertEqual( - ["a-library", "gke-backup", "new-library"], - sorted(config_change.get_changed_libraries()), - ) - - def test_get_qualified_commits_success(self): - baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" - latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" - gke_backup_commit = "b8691edb3f1d3c1583aa9cd89240eb359eebe9c7" - aiplatform_commit = "b82095baef02e525bee7bb1c48911c33b66acdf0" - network_management_commit = "efad09c9f0d46ae0786d810a88024363e06c6ca3" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="gke-backup", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/gkebackup/v1") - ], - ), - ConfigChangeTest.__get_a_library_config( - library_name="aiplatform", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/aiplatform/v1beta1") - ], - ), - ConfigChangeTest.__get_a_library_config( - library_name="network-management", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/networkmanagement/v1"), - GapicConfig( - proto_path="google/cloud/networkmanagement/v1beta1" - ), - ], - ), - ], - ), - ) - qualified_commits = config_change.get_qualified_commits() - self.assertEqual(3, len(qualified_commits)) - self.assertEqual({"gke-backup"}, qualified_commits[0].libraries) - self.assertEqual( - gke_backup_commit, - qualified_commits[0].commit.hexsha, - ) - self.assertEqual({"aiplatform"}, qualified_commits[1].libraries) - self.assertEqual( - aiplatform_commit, - qualified_commits[1].commit.hexsha, - ) - self.assertEqual({"network-management"}, qualified_commits[2].libraries) - self.assertEqual( - network_management_commit, - qualified_commits[2].commit.hexsha, - ) - - def test_get_qualified_commits_rest_numeric_enum_change_returns_a_qualified_commit( - self, - ): - baseline_commit = "45694d2bad602c52170096072d325aa644d550e5" - latest_commit = "758f0d1217d9c7fe398aa5efb1057ce4b6409e55" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="container", - gapic_configs=[GapicConfig(proto_path="google/container/v1")], - ) - ], - ), - ) - # one commit between latest_commit and baseline_commit which only - # changed BUILD.bazel. - # this commit changed `rest_numeric_enums`. - self.assertTrue(len(config_change.get_qualified_commits()) == 1) - - def test_get_qualified_commits_irrelevant_build_field_change_returns_empty_list( - self, - ): - baseline_commit = "bdda0174f68a738518ec311e05e6fd9bbe19cd78" - latest_commit = "c9a5050ef225b0011603e1109cf53ab1de0a8e53" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="chat", - gapic_configs=[GapicConfig(proto_path="google/chat/v1")], - ) - ], - ), - ) - # one commit between latest_commit and baseline_commit which only - # changed BUILD.bazel. - # this commit didn't change fields used in library generation. - self.assertTrue(len(config_change.get_qualified_commits()) == 0) - - def test_get_qualified_commits_add_build_file_returns_a_qualified_commit(self): - baseline_commit = "d007ca1b3cc820651530d44d5388533047ae1414" - latest_commit = "05d889e7dfe087fc2ddc9de9579f01d4e1c2f35e" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="cloudcontrolspartner", - gapic_configs=[ - GapicConfig( - proto_path="google/cloud/cloudcontrolspartner/v1" - ) - ], - ) - ], - ), - ) - # one commit between latest_commit and baseline_commit which added - # google/cloud/cloudcontrolspartner/v1. - self.assertTrue(len(config_change.get_qualified_commits()) == 1) - - @staticmethod - def __get_a_gen_config( - googleapis_commitish="", libraries: list[LibraryConfig] = None - ) -> GenerationConfig: - if libraries is None: - libraries = [] - return GenerationConfig( - gapic_generator_version="", - googleapis_commitish=googleapis_commitish, - libraries=libraries, - ) - - @staticmethod - def __get_a_library_config( - library_name: str, gapic_configs: list[GapicConfig] = None - ) -> LibraryConfig: - if gapic_configs is None: - gapic_configs = [] - return LibraryConfig( - api_shortname="existing_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=gapic_configs, - library_name=library_name, - ) diff --git a/sdk-platform-java/hermetic_build/common/tests/model/gapic_config_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/model/gapic_config_unit_tests.py deleted file mode 100644 index 864b2556e40f..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/model/gapic_config_unit_tests.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from common.model.gapic_config import GapicConfig - - -class GapicConfigTest(unittest.TestCase): - def test_get_version_returns_none(self): - self.assertIsNone(GapicConfig(proto_path="example/dir1/dir2").get_version()) - - def test_get_version_returns_non_stable_version(self): - self.assertEqual( - "v2p2beta1", - GapicConfig(proto_path="example/dir1/dir2/v2p2beta1").get_version(), - ) - self.assertEqual( - "v2beta1", - GapicConfig(proto_path="example/dir1/dir2/v2beta1").get_version(), - ) - - def test_get_version_returns_stable_version(self): - self.assertEqual( - "v20", - GapicConfig(proto_path="example/dir1/dir2/v20").get_version(), - ) - - def test_is_stable_with_no_version_returns_false(self): - self.assertFalse( - GapicConfig(proto_path="example/dir1/dir2/non_version").is_stable(), - ) - - def test_is_stable_with_non_stable_version_returns_false(self): - self.assertFalse( - GapicConfig(proto_path="example/dir1/dir2/v20alpha").is_stable(), - ) - self.assertFalse( - GapicConfig(proto_path="example/dir1/dir2/v20beta2").is_stable(), - ) - - def test_is_stable_with_stable_version_returns_true(self): - self.assertTrue( - GapicConfig(proto_path="example/dir1/dir2/v30").is_stable(), - ) - - def test_compare_configs_without_a_version(self): - config_len_3 = GapicConfig(proto_path="example/dir1/dir2") - config_len_4 = GapicConfig(proto_path="example/dir1/dir2/dir3") - self.assertLess( - config_len_3, - config_len_4, - "config_len_3 should be smaller since it has a lower depth.", - ) - - def test_compare_configs_only_one_has_a_stable_version(self): - versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v1") - non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3") - self.assertLess( - versioned_config, - non_versioned_config, - "versioned_config should be smaller since it has a version.", - ) - - def test_compare_configs_only_one_has_a_non_stable_version(self): - non_stable_versioned_config = GapicConfig( - proto_path="example/dir1/dir2/dir3/dir4/v1beta" - ) - non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3") - self.assertLess( - non_stable_versioned_config, - non_versioned_config, - "non_stable_versioned_config should be smaller since it has a version.", - ) - - def test_compare_configs_one_has_non_stable_and_one_has_stable_version(self): - stable_versioned_config = GapicConfig( - proto_path="example/dir1/dir2/dir3/dir4/v1" - ) - non_stable_versioned_config = GapicConfig(proto_path="example/dir1/dir2/v2beta") - self.assertLess( - stable_versioned_config, - non_stable_versioned_config, - "stable_versioned_config should be smaller since it has a stable version.", - ) - - def test_compare_configs_two_have_non_stable_version(self): - v3p2beta = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v3p2beta") - v2p4beta = GapicConfig(proto_path="example/dir1/dir2/v2p4beta") - self.assertLess( - v3p2beta, - v2p4beta, - "v3p2beta should be smaller since it has a higher version.", - ) - - def test_compare_configs_two_have_stable_version_different_depth(self): - v3 = GapicConfig(proto_path="example/dir1/dir2/v3") - v4 = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v4") - self.assertLess( - v3, - v4, - "v3 should be smaller since it has lower depth", - ) - - def test_compare_configs_two_have_stable_version_same_depth(self): - v4 = GapicConfig(proto_path="example/dir1/dir2/v4") - v3 = GapicConfig(proto_path="example/dir1/dir2/v3") - self.assertLess( - v4, - v3, - "v4 should be smaller since it has a higher version", - ) diff --git a/sdk-platform-java/hermetic_build/common/tests/model/gapic_inputs_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/model/gapic_inputs_unit_tests.py deleted file mode 100644 index 41e9e7d23347..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/model/gapic_inputs_unit_tests.py +++ /dev/null @@ -1,132 +0,0 @@ -import os -import unittest -from pathlib import Path - -from parameterized import parameterized -from common.model.gapic_inputs import parse - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -build_file = Path(os.path.join(resources_dir, "misc")).resolve() - - -class UtilitiesTest(unittest.TestCase): - @parameterized.expand( - [ - ("BUILD_no_additional_protos.bazel", " "), - ("BUILD_common_resources.bazel", " google/cloud/common_resources.proto"), - ("BUILD_comment_common_resources.bazel", " "), - ("BUILD_locations.bazel", " google/cloud/location/locations.proto"), - ("BUILD_comment_locations.bazel", " "), - ("BUILD_iam_policy.bazel", " google/iam/v1/iam_policy.proto"), - ("BUILD_comment_iam_policy.bazel", " "), - ( - "BUILD_iam_locations.bazel", - " google/cloud/location/locations.proto google/iam/v1/iam_policy.proto", - ), - ] - ) - def test_gapic_inputs_parse_additional_protos(self, build_name, expected): - parsed = parse(build_file, "", build_name) - self.assertEqual( - expected, - parsed.additional_protos, - ) - - def test_gapic_inputs_parse_grpc_only_succeeds(self): - parsed = parse(build_file, "", "BUILD_grpc.bazel") - self.assertEqual("grpc", parsed.transport) - - def test_gapic_inputs_parse_grpc_rest_succeeds(self): - parsed = parse(build_file, "", "BUILD_grpc_rest.bazel") - self.assertEqual("grpc+rest", parsed.transport) - - def test_gapic_inputs_parse_rest_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest.bazel") - self.assertEqual("rest", parsed.transport) - - def test_gapic_inputs_parse_empty_include_samples_succeeds(self): - parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel") - self.assertEqual("false", parsed.include_samples) - - def test_gapic_inputs_parse_include_samples_false_succeeds(self): - parsed = parse(build_file, "", "BUILD_include_samples_false.bazel") - self.assertEqual("false", parsed.include_samples) - - def test_gapic_inputs_parse_include_samples_true_succeeds(self): - parsed = parse(build_file, "", "BUILD_include_samples_true.bazel") - self.assertEqual("true", parsed.include_samples) - - def test_gapic_inputs_parse_empty_rest_numeric_enums_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest_numeric_enums_empty.bazel") - self.assertEqual("false", parsed.rest_numeric_enum) - - def test_gapic_inputs_parse_rest_numeric_enums_false_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest_numeric_enums_false.bazel") - self.assertEqual("false", parsed.rest_numeric_enum) - - def test_gapic_inputs_parse_rest_numeric_enums_true_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest_numeric_enums_true.bazel") - self.assertEqual("true", parsed.rest_numeric_enum) - - def test_gapic_inputs_parse_no_gapic_library_returns_proto_only_true(self): - # include_samples_empty only has a gradle assembly rule - parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel") - self.assertEqual("true", parsed.proto_only) - - def test_gapic_inputs_parse_with_gapic_library_returns_proto_only_false(self): - # rest.bazel has a java_gapic_library rule - parsed = parse(build_file, "", "BUILD_rest.bazel") - self.assertEqual("false", parsed.proto_only) - - def test_gapic_inputs_parse_gapic_yaml_succeeds(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_gapic_yaml.bazel") - self.assertEqual("test/versioned/path/test_gapic_yaml.yaml", parsed.gapic_yaml) - - def test_gapic_inputs_parse_no_gapic_yaml_returns_empty_string(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_no_gapic_yaml.bazel") - self.assertEqual("", parsed.gapic_yaml) - - def test_gapic_inputs_parse_service_config_succeeds(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_service_config.bazel") - self.assertEqual( - "test/versioned/path/test_service_config.json", parsed.service_config - ) - - def test_gapic_inputs_parse_service_yaml_relative_target(self): - parsed = parse( - build_file, - "google/cloud/compute/v1", - "BUILD_service_config_relative_target.bazel", - ) - self.assertEqual( - "google/cloud/compute/v1/compute_grpc_service_config.json", - parsed.service_config, - ) - - def test_gapic_inputs_parse_no_service_config_returns_empty_string(self): - parsed = parse( - build_file, "test/versioned/path", "BUILD_no_service_config.bazel" - ) - self.assertEqual("", parsed.service_config) - - def test_gapic_inputs_parse_service_yaml_succeeds(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_service_yaml.bazel") - self.assertEqual( - "test/versioned/path/test_service_yaml.yaml", parsed.service_yaml - ) - - def test_gapic_inputs_parse_service_yaml_absolute_target(self): - parsed = parse(build_file, "", "BUILD_service_yaml_absolute_target.bazel") - self.assertEqual( - "google/cloud/videointelligence/videointelligence_v1p3beta1.yaml", - parsed.service_yaml, - ) - - def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_no_service_yaml.bazel") - self.assertEqual("", parsed.service_yaml) - - def test_gapic_inputs_parse_proto_only_returns_grpc(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_proto_only.bazel") - self.assertEqual("grpc", parsed.transport) diff --git a/sdk-platform-java/hermetic_build/common/tests/model/generation_config_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/model/generation_config_unit_tests.py deleted file mode 100644 index 2b89a1acc89e..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/model/generation_config_unit_tests.py +++ /dev/null @@ -1,258 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from pathlib import Path -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() - -library_1 = LibraryConfig( - api_shortname="a_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], -) -library_2 = LibraryConfig( - api_shortname="another_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], -) -common_protos_library = LibraryConfig( - api_shortname="common-protos", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], -) - - -class GenerationConfigTest(unittest.TestCase): - def test_generation_config_default_value(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[], - ) - self.assertEqual("", config.libraries_bom_version) - - def test_generation_config_with_generator_version_env_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Environment variable GENERATOR_VERSION is not set", - GenerationConfig, - googleapis_commitish="", - libraries=[], - ) - - def test_generation_config_set_generator_version_from_env(self): - os.environ["GENERATOR_VERSION"] = "1.2.3" - config = GenerationConfig( - googleapis_commitish="", - libraries=[], - ) - self.assertEqual("1.2.3", config.gapic_generator_version) - os.environ.pop("GENERATOR_VERSION") - - def test_from_yaml_succeeds(self): - config = GenerationConfig.from_yaml(f"{test_config_dir}/generation_config.yaml") - self.assertEqual("2.34.0", config.gapic_generator_version) - self.assertEqual( - "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish - ) - self.assertEqual("26.37.0", config.libraries_bom_version) - library = config.libraries[0] - self.assertEqual("cloudasset", library.api_shortname) - self.assertEqual("Cloud Asset Inventory", library.name_pretty) - self.assertEqual( - "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - library.product_documentation, - ) - self.assertEqual( - "provides inventory services based on a time series database.", - library.api_description, - ) - self.assertEqual("asset", library.library_name) - self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team) - self.assertEqual( - "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1", - library.excluded_poms, - ) - self.assertEqual("google-iam-policy", library.excluded_dependencies) - gapics = library.gapic_configs - self.assertEqual(5, len(gapics)) - self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path) - self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path) - self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path) - self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path) - self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) - - def test_get_proto_path_to_library_name_success(self): - paths = GenerationConfig.from_yaml( - f"{test_config_dir}/generation_config.yaml" - ).get_proto_path_to_library_name() - self.assertEqual( - { - "google/cloud/asset/v1": "asset", - "google/cloud/asset/v1p1beta1": "asset", - "google/cloud/asset/v1p2beta1": "asset", - "google/cloud/asset/v1p5beta1": "asset", - "google/cloud/asset/v1p7beta1": "asset", - }, - paths, - ) - - def test_is_monorepo_with_one_library_returns_false(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1], - ) - self.assertFalse(config.is_monorepo()) - - def test_is_monorepo_with_two_libraries_returns_true(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1, library_2], - ) - self.assertTrue(config.is_monorepo()) - - def test_contains_common_protos_with_common_protos_returns_true(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1, library_2, common_protos_library], - ) - self.assertTrue(config.contains_common_protos()) - - def test_contains_common_protos_without_common_protos_returns_false(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1, library_2], - ) - self.assertFalse(config.contains_common_protos()) - - def test_validate_with_duplicate_library_name_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "the same library name", - GenerationConfig, - gapic_generator_version="", - googleapis_commitish="", - libraries=[ - LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret API", - product_documentation="", - api_description="", - gapic_configs=list(), - ), - LibraryConfig( - api_shortname="another-secret", - name_pretty="Another Secret API", - product_documentation="", - api_description="", - gapic_configs=list(), - library_name="secretmanager", - ), - ], - ) - - def test_from_yaml_without_googleapis_commitish_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Repo level parameter, googleapis_commitish", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_googleapis.yaml", - ) - - def test_from_yaml_without_libraries_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Repo level parameter, libraries", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_libraries.yaml", - ) - - def test_from_yaml_without_api_shortname_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Library level parameter, api_shortname", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_api_shortname.yaml", - ) - - def test_from_yaml_without_api_description_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"Library level parameter, api_description.*'api_shortname': 'apigeeconnect'.*", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_api_description.yaml", - ) - - def test_from_yaml_without_name_pretty_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"Library level parameter, name_pretty.*'api_shortname': 'apigeeconnect'.*", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_name_pretty.yaml", - ) - - def test_from_yaml_without_product_documentation_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"Library level parameter, product_documentation.*'api_shortname': 'apigeeconnect'.*", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_product_docs.yaml", - ) - - def test_from_yaml_without_gapics_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Library level parameter, GAPICs.*'api_shortname': 'apigeeconnect'.*", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_gapics_key.yaml", - ) - - def test_from_yaml_without_proto_path_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "GAPIC level parameter, proto_path", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_proto_path.yaml", - ) - - def test_from_yaml_with_zero_library_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Library is None", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_library_value.yaml", - ) - - def test_from_yaml_with_zero_proto_path_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"GAPICs is None in.*'api_shortname': 'apigeeconnect'.*", - GenerationConfig.from_yaml, - f"{test_config_dir}/config_without_gapics_value.yaml", - ) diff --git a/sdk-platform-java/hermetic_build/common/tests/model/library_config_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/model/library_config_unit_tests.py deleted file mode 100644 index 4935979ffad2..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/model/library_config_unit_tests.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from common.model.gapic_config import GapicConfig -from common.model.library_config import LibraryConfig - - -class LibraryConfigTest(unittest.TestCase): - def test_get_library_returns_library_name(self): - library = LibraryConfig( - api_shortname="secret", - name_pretty="", - product_documentation="", - api_description="", - gapic_configs=list(), - library_name="secretmanager", - ) - self.assertEqual("secretmanager", library.get_library_name()) - - def test_get_library_returns_api_shortname(self): - library = LibraryConfig( - api_shortname="secret", - name_pretty="", - product_documentation="", - api_description="", - gapic_configs=list(), - ) - self.assertEqual("secret", library.get_library_name()) - - def test_get_sorted_gapic_configs_returns_correct_order(self): - v1beta1 = GapicConfig(proto_path="google/spanner/v1beta1") - v1 = GapicConfig(proto_path="google/spanner/v1") - v1alpha1 = GapicConfig(proto_path="google/spanner/v1alpha") - v2 = GapicConfig(proto_path="google/spanner/v2") - admin_v2 = GapicConfig(proto_path="google/spanner/admin/v2") - non_versioned = GapicConfig(proto_path="google/spanner/type") - library = LibraryConfig( - api_shortname="secret", - name_pretty="", - product_documentation="", - api_description="", - gapic_configs=[v1alpha1, v1, v2, admin_v2, non_versioned, v1beta1], - ) - self.assertEqual( - [ - v2, - v1, - admin_v2, - v1beta1, - v1alpha1, - non_versioned, - ], - library.get_sorted_gapic_configs(), - ) - - def test_init_invalid_distribution_name_raise_value_error(self): - self.assertRaisesRegex( - ValueError, - "fake-distribution-name is not a valid distribution name.", - LibraryConfig, - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - distribution_name="fake-distribution-name", - ) - - def test_get_distribution_name_cloud_api(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - ) - self.assertEqual( - "com.google.cloud:google-cloud-baremetalsolution", - library.get_maven_coordinate(), - ) - self.assertEqual("google-cloud-baremetalsolution", library.get_artifact_id()) - - def test_get_distribution_name_non_cloud_api(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - cloud_api=False, - group_id="com.example", - ) - self.assertEqual( - "com.example:google-baremetalsolution", library.get_maven_coordinate() - ) - self.assertEqual("google-baremetalsolution", library.get_artifact_id()) - - def test_get_distribution_name_with_distribution_name(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - distribution_name="com.example:baremetalsolution", - ) - self.assertEqual( - "com.example:baremetalsolution", library.get_maven_coordinate() - ) - self.assertEqual("baremetalsolution", library.get_artifact_id()) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/cli/empty_config.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/cli/empty_config.yaml deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_additional_protos_in_gapic.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_additional_protos_in_gapic.bazel deleted file mode 100644 index cb124dbbc741..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_additional_protos_in_gapic.bazel +++ /dev/null @@ -1,6 +0,0 @@ -java_gapic_library( - deps = [ - "//google/cloud/location:location_java_proto", - "//google/iam/v1:iam_java_proto" - ], -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_common_resources.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_common_resources.bazel deleted file mode 100644 index 126ffdb7cae9..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_common_resources.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - #"//google/cloud:common_resources_proto", - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_iam_policy.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_iam_policy.bazel deleted file mode 100644 index a9a2c1ca7587..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_iam_policy.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - # "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_locations.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_locations.bazel deleted file mode 100644 index 8b96e3ab8195..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_comment_locations.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - # "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_common_resources.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_common_resources.bazel deleted file mode 100644 index 9b749e6ad5f9..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_common_resources.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud:common_resources_proto", - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_gapic_yaml.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_gapic_yaml.bazel deleted file mode 100644 index b55f4550d8ca..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_gapic_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - gapic_yaml = "test_gapic_yaml.yaml", -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_grpc.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_grpc.bazel deleted file mode 100644 index 3e59a9bd355d..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_grpc.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "grpc", -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_grpc_rest.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_grpc_rest.bazel deleted file mode 100644 index 8a98e7e646c3..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_grpc_rest.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "grpc+rest", -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_iam_locations.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_iam_locations.bazel deleted file mode 100644 index d0c971da7c70..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_iam_locations.bazel +++ /dev/null @@ -1,6 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_iam_policy.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_iam_policy.bazel deleted file mode 100644 index af5d4a32f848..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_iam_policy.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_empty.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_empty.bazel deleted file mode 100644 index b9d4cd56e0e1..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_empty.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_false.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_false.bazel deleted file mode 100644 index 8e95c3d2a628..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_false.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - include_samples = False, -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_true.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_true.bazel deleted file mode 100644 index bac72b678f25..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_include_samples_true.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - include_samples = True, -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_locations.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_locations.bazel deleted file mode 100644 index 29ee14fdba07..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_locations.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_additional_protos.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_additional_protos.bazel deleted file mode 100644 index a22257cad49d..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_additional_protos.bazel +++ /dev/null @@ -1,4 +0,0 @@ -proto_library_with_info( - deps = [ - ] -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_gapic_yaml.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_gapic_yaml.bazel deleted file mode 100644 index 1e9462aa303e..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_gapic_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - gapic_yaml = None -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_service_config.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_service_config.bazel deleted file mode 100644 index dbde6de05c25..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_service_config.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = None -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_service_yaml.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_service_yaml.bazel deleted file mode 100644 index 05bae16d5def..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_no_service_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = None -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_proto_only.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_proto_only.bazel deleted file mode 100644 index 26bcea612643..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_proto_only.bazel +++ /dev/null @@ -1,16 +0,0 @@ -java_gapic_assembly_gradle_pkg( - name = "google-api-java", - transport = "grpc+rest", - deps = [ - "annotations_proto", - "auth_proto", - "backend_proto", - "billing_proto", - "client_proto", - "config_change_proto", - "consumer_proto", - "context_proto", - "control_proto", - "distribution_proto", - ], -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest.bazel deleted file mode 100644 index 9dff694297ca..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "rest", -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel deleted file mode 100644 index 992b91e52ce0..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel deleted file mode 100644 index a446c6b2a73a..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - rest_numeric_enums = False -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel deleted file mode 100644 index c4d7fefeb4bc..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - rest_numeric_enums = True -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_config.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_config.bazel deleted file mode 100644 index 097d1bb6bddc..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_config.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = "test_service_config.json" -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_config_relative_target.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_config_relative_target.bazel deleted file mode 100644 index ccd59af2fb85..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_config_relative_target.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = ":compute_grpc_service_config.json" -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_yaml.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_yaml.bazel deleted file mode 100644 index f7e4c91f4ed0..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = "test_service_yaml.yaml" -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel deleted file mode 100644 index ded899dff74e..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = "//google/cloud/videointelligence:videointelligence_v1p3beta1.yaml", -) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_target_with_nested_parenthesis.bazel b/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_target_with_nested_parenthesis.bazel deleted file mode 100644 index 49d7dba943bb..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/BUILD_target_with_nested_parenthesis.bazel +++ /dev/null @@ -1,13 +0,0 @@ -proto_library_with_info( - # this comment should not indicate an end of a pattern match ) - deps = [ - "//google/cloud/location:location_proto", - ] -) - -java_gapic_library( - # this comment should not indicate an end of a pattern match ) - deps = [ - "//google/iam/v1:iam_java_proto" - ], -) diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/TESTWORKSPACE b/sdk-platform-java/hermetic_build/common/tests/resources/misc/TESTWORKSPACE deleted file mode 100644 index 60b3036d9daf..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/TESTWORKSPACE +++ /dev/null @@ -1,133 +0,0 @@ -# test workspace file obtained from sdk-platform-java - -workspace(name = "gapic_generator_java") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -# gax-java and its transitive dependencies must be imported before -# gapic-generator-java dependencies to match the order in googleapis repository, -# which in its turn, prioritizes actual generated clients runtime dependencies -# over the generator dependencies. -local_repository( - name = "com_google_api_gax_java", - path = "gax-java", -) - -load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties") - -com_google_api_gax_java_properties( - name = "com_google_api_gax_java_properties", - file = "@com_google_api_gax_java//:dependencies.properties", -) - -load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories") - -com_google_api_gax_java_repositories() - -_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d" - -http_archive( - name = "com_google_googleapis", - strip_prefix = "googleapis-%s" % _googleapis_commit, - urls = [ - "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit, - ], -) - -# protobuf -RULES_JVM_EXTERNAL_TAG = "4.5" - -RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6" - -http_archive( - name = "rules_jvm_external", - sha256 = RULES_JVM_EXTERNAL_SHA, - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, - url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, -) - -load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") - -rules_jvm_external_deps() - -load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") - -rules_jvm_external_setup() - -load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") -load("@rules_jvm_external//:defs.bzl", "maven_install") - -maven_install( - artifacts = PROTOBUF_MAVEN_ARTIFACTS, - repositories = ["https://repo.maven.apache.org/maven2/"], -) - -_gapic_generator_java_version = "2.25.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} - -maven_install( - artifacts = [ - "com.google.api:gapic-generator-java:" + _gapic_generator_java_version, - ], - fail_on_missing_checksum = False, - repositories = [ - "m2Local", - "https://repo.maven.apache.org/maven2/", - ], -) - -protobuf_deps() - -# Bazel rules. -_rules_gapic_version = "0.5.5" - -http_archive( - name = "rules_gapic", - strip_prefix = "rules_gapic-%s" % _rules_gapic_version, - urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version], -) - -# Java dependencies. -load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") - -switched_rules_by_language( - name = "com_google_googleapis_imports", - gapic = True, - grpc = True, - java = True, -) - -load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories") - -grpc_java_repositories() - -_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3" - -http_archive( - name = "com_google_disco_to_proto3_converter", - strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit, - urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit], -) - -# Showcase -_showcase_version = "0.28.2" - -http_archive( - name = "com_google_gapic_showcase", - strip_prefix = "gapic-showcase-%s" % _showcase_version, - urls = [ - "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version, - ], -) - -http_archive( - name = "rules_pkg", - sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", - ], -) - -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") - -rules_pkg_dependencies() diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/misc/versions.txt b/sdk-platform-java/hermetic_build/common/tests/resources/misc/versions.txt deleted file mode 100644 index e4258504e423..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/misc/versions.txt +++ /dev/null @@ -1,9 +0,0 @@ -# test versions.txt obtained from sdk-platform-java - -gapic-generator-java:2.25.0:2.25.1-SNAPSHOT -api-common:2.16.0:2.16.1-SNAPSHOT -gax:2.33.0:2.33.1-SNAPSHOT -gax-grpc:2.34.0:2.33.1-SNAPSHOT -gax-httpjson:0.118.0:0.118.1-SNAPSHOT -proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT -grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_api_description.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_api_description.yaml deleted file mode 100644 index 79ff1350678c..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_api_description.yaml +++ /dev/null @@ -1,5 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_api_shortname.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_api_shortname.yaml deleted file mode 100644 index ec8206be615e..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_api_shortname.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_gapics_key.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_gapics_key.yaml deleted file mode 100644 index 739a4d92398e..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_gapics_key.yaml +++ /dev/null @@ -1,3 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_gapics_value.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_gapics_value.yaml deleted file mode 100644 index ec49e4a66902..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_gapics_value.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - GAPICs: diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_googleapis.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_googleapis.yaml deleted file mode 100644 index e5a00ca4ee3e..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_googleapis.yaml +++ /dev/null @@ -1,8 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_libraries.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_libraries.yaml deleted file mode 100644 index dbbe2ea318bc..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_libraries.yaml +++ /dev/null @@ -1 +0,0 @@ -gapic_generator_version: 2.34.0 diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_library_value.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_library_value.yaml deleted file mode 100644 index 174a293000cc..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_library_value.yaml +++ /dev/null @@ -1,2 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_name_pretty.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_name_pretty.yaml deleted file mode 100644 index f8612ad9ca65..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_name_pretty.yaml +++ /dev/null @@ -1,6 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - api_description: "allows the Apigee hybrid management" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_product_docs.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_product_docs.yaml deleted file mode 100644 index e3921d2c0d25..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_product_docs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_proto_path.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_proto_path.yaml deleted file mode 100644 index e37b0cef6383..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/config_without_proto_path.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - GAPICs: - - random_key: diff --git a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/generation_config.yaml b/sdk-platform-java/hermetic_build/common/tests/resources/test-config/generation_config.yaml deleted file mode 100644 index 94fb1c83379c..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/resources/test-config/generation_config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - codeowner_team: "@googleapis/analytics-dpe" - excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 - excluded_dependencies: google-iam-policy - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 diff --git a/sdk-platform-java/hermetic_build/common/tests/utils/__init__.py b/sdk-platform-java/hermetic_build/common/tests/utils/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py deleted file mode 100644 index 4d45c6531c4b..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from common.model.gapic_config import GapicConfig -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig -from common.utils.generation_config_comparator import ChangeType -from common.utils.generation_config_comparator import compare_config - - -class GenerationConfigComparatorTest(unittest.TestCase): - def setUp(self) -> None: - self.baseline_library = LibraryConfig( - api_shortname="existing_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], - ) - self.current_library = LibraryConfig( - api_shortname="existing_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], - ) - self.baseline_config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[self.baseline_library], - ) - self.current_config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[self.current_library], - ) - - def test_compare_config_not_change(self): - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue(len(result.change_to_libraries) == 0) - - def test_compare_config_googleapis_update(self): - self.baseline_config.googleapis_commitish = ( - "1a45bf7393b52407188c82e63101db7dc9c72026" - ) - self.current_config.googleapis_commitish = ( - "1e6517ef4f949191c9e471857cf5811c8abcab84" - ) - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result.change_to_libraries) - - def test_compare_config_generator_update(self): - self.baseline_config.gapic_generator_version = "1.2.3" - self.current_config.gapic_generator_version = "1.2.4" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("gapic_generator_version", config_change.changed_param) - self.assertEqual("1.2.4", config_change.current_value) - - def test_compare_config_libraries_bom_update(self): - self.baseline_config.libraries_bom_version = "26.36.0" - self.current_config.libraries_bom_version = "26.37.0" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("libraries_bom_version", config_change.changed_param) - self.assertEqual("26.37.0", config_change.current_value) - - def test_compare_protobuf_update(self): - self.baseline_config.protoc_version = "3.25.2" - self.current_config.protoc_version = "3.27.0" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("protoc_version", config_change.changed_param) - self.assertEqual("3.27.0", config_change.current_value) - - def test_compare_config_grpc_update(self): - self.baseline_config.grpc_version = "1.60.0" - self.current_config.grpc_version = "1.61.0" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("grpc_version", config_change.changed_param) - self.assertEqual("1.61.0", config_change.current_value) - - def test_compare_config_template_excludes_update(self): - self.baseline_config.template_excludes = [".github/*", ".kokoro/*"] - self.current_config.template_excludes = [ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - ] - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("template_excludes", config_change.changed_param) - self.assertEqual( - [ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - ], - config_change.current_value, - ) - - def test_compare_config_library_addition(self): - self.current_config.libraries.append( - LibraryConfig( - api_shortname="new_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], - ) - ) - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] - self.assertEqual("new_library", config_change.library_name) - - def test_compare_config_library_removal_does_not_have_repo_or_library_level_change( - self, - ): - self.current_config.libraries = [] - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 0 - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 0 - ) - - def test_compare_config_api_shortname_update_without_library_name(self): - self.current_config.libraries[0].api_shortname = "new_api_shortname" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] - self.assertEqual("new_api_shortname", config_change.library_name) - - def test_compare_config_api_shortname_update_with_library_name_raise_error(self): - self.baseline_config.libraries[0].library_name = "old_library_name" - self.current_config.libraries[0].library_name = "old_library_name" - self.current_config.libraries[0].api_shortname = "new_api_shortname" - self.assertRaisesRegex( - ValueError, - r"api_shortname.*library_name", - compare_config, - self.baseline_config, - self.current_config, - ) - - def test_compare_config_library_name_update(self): - self.current_config.libraries[0].library_name = "new_library_name" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] - self.assertEqual("new_library_name", config_change.library_name) - - def test_compare_config_api_description_update(self): - self.current_config.libraries[0].api_description = "updated description" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("api_description", config_change.changed_param) - self.assertEqual("updated description", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_name_pretty_update(self): - self.current_config.libraries[0].name_pretty = "new name" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("name_pretty", config_change.changed_param) - self.assertEqual("new name", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_product_docs_update(self): - self.current_config.libraries[0].product_documentation = "new docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("product_documentation", config_change.changed_param) - self.assertEqual("new docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_library_type_update(self): - self.current_config.libraries[0].library_type = "GAPIC_COMBO" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("library_type", config_change.changed_param) - self.assertEqual("GAPIC_COMBO", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_release_level_update(self): - self.current_config.libraries[0].release_level = "STABLE" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("release_level", config_change.changed_param) - self.assertEqual("STABLE", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_api_id_update(self): - self.current_config.libraries[0].api_id = "new_id" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("api_id", config_change.changed_param) - self.assertEqual("new_id", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_api_reference_update(self): - self.current_config.libraries[0].api_reference = "new api_reference" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("api_reference", config_change.changed_param) - self.assertEqual("new api_reference", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_code_owner_team_update(self): - self.current_config.libraries[0].codeowner_team = "new team" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("codeowner_team", config_change.changed_param) - self.assertEqual("new team", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_excluded_deps_update(self): - self.current_config.libraries[0].excluded_dependencies = "group:artifact" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("excluded_dependencies", config_change.changed_param) - self.assertEqual("group:artifact", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_excluded_poms_update(self): - self.current_config.libraries[0].excluded_poms = "pom.xml" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("excluded_poms", config_change.changed_param) - self.assertEqual("pom.xml", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_client_docs_update(self): - self.current_config.libraries[0].client_documentation = "new client docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("client_documentation", config_change.changed_param) - self.assertEqual("new client docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_distribution_name_update(self): - self.current_config.libraries[0].distribution_name = "new_group:new_artifact" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("distribution_name", config_change.changed_param) - self.assertEqual("new_group:new_artifact", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_group_id_update(self): - self.current_config.libraries[0].group_id = "new_group" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("group_id", config_change.changed_param) - self.assertEqual("new_group", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_issue_tracker_update(self): - self.current_config.libraries[0].issue_tracker = "new issue tracker" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("issue_tracker", config_change.changed_param) - self.assertEqual("new issue tracker", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_rest_docs_update(self): - self.current_config.libraries[0].rest_documentation = "new rest docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("rest_documentation", config_change.changed_param) - self.assertEqual("new rest docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_rpc_docs_update(self): - self.current_config.libraries[0].rpc_documentation = "new rpc docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("rpc_documentation", config_change.changed_param) - self.assertEqual("new rpc docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_cloud_api_update(self): - self.current_config.libraries[0].cloud_api = False - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("cloud_api", config_change.changed_param) - self.assertEqual(False, config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_requires_billing_update(self): - self.current_config.libraries[0].requires_billing = False - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("requires_billing", config_change.changed_param) - self.assertEqual(False, config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_extra_versioned_mod_update(self): - self.current_config.libraries[0].extra_versioned_modules = "extra module" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("extra_versioned_modules", config_change.changed_param) - self.assertEqual("extra module", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_version_addition(self): - self.current_config.libraries[0].gapic_configs = [ - GapicConfig(proto_path="google/new/library/v1") - ] - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue(len(result.change_to_libraries[ChangeType.GAPIC_ADDITION]) == 1) - config_change = result.change_to_libraries[ChangeType.GAPIC_ADDITION][0] - self.assertEqual("", config_change.changed_param) - self.assertEqual("google/new/library/v1", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) diff --git a/sdk-platform-java/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py b/sdk-platform-java/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py deleted file mode 100644 index 80d7797571f8..000000000000 --- a/sdk-platform-java/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from pathlib import Path -from common.utils.proto_path_utils import find_versioned_proto_path -from common.utils.proto_path_utils import ends_with_version - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() - - -class ProtoPathsUtilsTest(unittest.TestCase): - def test_find_versioned_proto_path_nested_version_success(self): - proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" - expected = "google/cloud/aiplatform/v1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_path_success(self): - proto_path = "google/cloud/asset/v1p2beta1/assets.proto" - expected = "google/cloud/asset/v1p2beta1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_without_version_return_itself(self): - proto_path = "google/type/color.proto" - expected = "google/type/color.proto" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_ends_with_version_valid(self): - self.assertTrue(ends_with_version("google/cloud/gsuiteaddons/v1")) - self.assertTrue(ends_with_version("google/iam/v1beta")) - self.assertTrue(ends_with_version("google/iam/v2betav1")) - self.assertTrue(ends_with_version("google/cloud/alloydb/connectors/v1alpha")) - self.assertTrue(ends_with_version("v1")) - self.assertTrue(ends_with_version("anything/v123")) - - def test_ends_with_version_invalid(self): - self.assertFalse(ends_with_version("google/apps/script/type")) - self.assertFalse(ends_with_version("google/apps/script/type/docs")) - self.assertFalse( - ends_with_version("google/cloud/alloydb/connectors/v123/something") - ) - self.assertFalse(ends_with_version("")) - self.assertFalse(ends_with_version("noVersion")) diff --git a/sdk-platform-java/hermetic_build/common/utils/generation_config_comparator.py b/sdk-platform-java/hermetic_build/common/utils/generation_config_comparator.py deleted file mode 100644 index 8274af207476..000000000000 --- a/sdk-platform-java/hermetic_build/common/utils/generation_config_comparator.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections import defaultdict -from typing import Any -from typing import Dict -from typing import List -from common.model.config_change import ChangeType -from common.model.config_change import ConfigChange -from common.model.config_change import LibraryChange -from common.model.config_change import HashLibrary -from common.model.gapic_config import GapicConfig -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig - - -def compare_config( - baseline_config: GenerationConfig, - current_config: GenerationConfig, - force_regenerate_all: bool = False, -) -> ConfigChange: - """ - Compare two GenerationConfig object and output a mapping from ConfigChange - to a list of ConfigChange objects. - All libraries in the current configuration will be affected if one of the - repository level parameters is changed. - - :param baseline_config: the baseline GenerationConfig object - :param current_config: the current GenerationConfig object - :return: a ConfigChange objects. - """ - diff = defaultdict(list[LibraryChange]) - # Exclude `libraries` because an empty library list (e.g., by library - # removal) will cause this parameter appears in the sorted param list, - # which leads to unequal list of parameters. - excluded_params = {"libraries"} - baseline_params = __convert_params_to_sorted_list( - obj=baseline_config, excluded_params=excluded_params - ) - current_params = __convert_params_to_sorted_list( - obj=current_config, excluded_params=excluded_params - ) - if force_regenerate_all: - config_change = LibraryChange( - changed_param="force_regenerate_all", - current_value="true", - ) - diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) - - for baseline_param, current_param in zip(baseline_params, current_params): - if baseline_param == current_param: - continue - if baseline_param[0] == "googleapis_commitish": - diff[ChangeType.GOOGLEAPIS_COMMIT] = [] - else: - config_change = LibraryChange( - changed_param=current_param[0], - current_value=current_param[1], - ) - diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) - __compare_libraries( - diff=diff, - baseline_library_configs=baseline_config.libraries, - current_library_configs=current_config.libraries, - ) - return ConfigChange( - change_to_libraries=diff, - baseline_config=baseline_config, - current_config=current_config, - ) - - -def __compare_libraries( - diff: Dict[ChangeType, list[LibraryChange]], - baseline_library_configs: List[LibraryConfig], - current_library_configs: List[LibraryConfig], -) -> None: - """ - Compare two lists of LibraryConfig and put the difference into a - given Dict. - - :param diff: a mapping from ConfigChange to a list of ConfigChange objects. - :param baseline_library_configs: a list of LibraryConfig object. - :param current_library_configs: a list of LibraryConfig object. - """ - baseline_libraries = __convert_to_hashed_library_dict(baseline_library_configs) - current_libraries = __convert_to_hashed_library_dict(current_library_configs) - changed_libraries = [] - # 1st round comparison. - for library_name, hash_library in baseline_libraries.items(): - # 1. find any library removed from baseline_libraries. - # a library is removed from baseline_libraries if the library_name - # is not in current_libraries. - # please see the reason of comment out these lines of code in the - # comment of ChangeType.LIBRARIES_REMOVAL. - # if library_name not in current_libraries: - # config_change = ConfigChange( - # changed_param="", current_value="", library_name=library_name - # ) - # diff[ChangeType.LIBRARIES_REMOVAL].append(config_change) - - # 2. find any library that exists in both configs but at least one - # parameter is changed, which means the hash value is different. - if ( - library_name in current_libraries - and hash_library.hash_value != current_libraries[library_name].hash_value - ): - changed_libraries.append(library_name) - # 2nd round comparison. - for library_name in current_libraries: - # find any library added to current_libraries. - # a library is added to current_libraries if the library_name - # is not in baseline_libraries. - if library_name not in baseline_libraries: - config_change = LibraryChange( - changed_param="", current_value="", library_name=library_name - ) - diff[ChangeType.LIBRARIES_ADDITION].append(config_change) - # 3rd round comparison. - __compare_changed_libraries( - diff=diff, - baseline_libraries=baseline_libraries, - current_libraries=current_libraries, - changed_libraries=changed_libraries, - ) - - -def __convert_to_hashed_library_dict( - libraries: List[LibraryConfig], -) -> Dict[str, HashLibrary]: - """ - Convert a list of LibraryConfig objects to a Dict. - For each library object, the key is the library_name of the object, the - value is a HashLibrary object. - - :param libraries: a list of LibraryConfig object. - :return: a mapping from library_name to HashLibrary object. - """ - return { - library.get_library_name(): HashLibrary(hash(library), library) - for library in libraries - } - - -def __compare_changed_libraries( - diff: Dict[ChangeType, list[LibraryChange]], - baseline_libraries: Dict[str, HashLibrary], - current_libraries: Dict[str, HashLibrary], - changed_libraries: List[str], -) -> None: - """ - Compare each library with the same library_name to find what parameters are - changed. - - :param diff: a mapping from ConfigChange to a list of ConfigChange objects. - :param baseline_libraries: a mapping from library_name to HashLibrary - object. - :param current_libraries: a mapping from library_name to HashLibrary object. - :param changed_libraries: a list of library_name of changed libraries. - :raise ValueError: if api_shortname of a library is changed but library_name - remains the same. - """ - for library_name in changed_libraries: - baseline_library = baseline_libraries[library_name].library - current_library = current_libraries[library_name].library - baseline_params = __convert_params_to_sorted_list(baseline_library) - current_params = __convert_params_to_sorted_list(current_library) - for baseline_param, current_param in zip(baseline_params, current_params): - if baseline_param == current_param: - continue - if baseline_param[0] == "api_shortname": - raise ValueError( - f"{library_name}: api_shortname must not change when library_name remains the same." - ) - else: - config_change = LibraryChange( - changed_param=current_param[0], - current_value=current_param[1], - library_name=library_name, - ) - diff[ChangeType.LIBRARY_LEVEL_CHANGE].append(config_change) - - # compare gapic_configs - baseline_gapic_configs = baseline_library.gapic_configs - current_gapic_configs = current_library.gapic_configs - __compare_gapic_configs( - diff=diff, - library_name=library_name, - baseline_gapic_configs=baseline_gapic_configs, - current_gapic_configs=current_gapic_configs, - ) - - -def __compare_gapic_configs( - diff: Dict[ChangeType, list[LibraryChange]], - library_name: str, - baseline_gapic_configs: List[GapicConfig], - current_gapic_configs: List[GapicConfig], -) -> None: - baseline_proto_paths = {config.proto_path for config in baseline_gapic_configs} - current_proto_paths = {config.proto_path for config in current_gapic_configs} - # 1st round of comparison, find any versioned proto_path is removed - # from baseline gapic configs. - # please see the reason of comment out these lines of code in the - # comment of ChangeType.GAPIC_REMOVAL. - # for proto_path in baseline_proto_paths: - # if proto_path in current_proto_paths: - # continue - # config_change = ConfigChange( - # changed_param="", current_value=proto_path, library_name=library_name - # ) - # diff[ChangeType.GAPIC_REMOVAL].append(config_change) - - # 2nd round of comparison, find any versioned proto_path is added - # to current gapic configs. - for proto_path in current_proto_paths: - if proto_path in baseline_proto_paths: - continue - config_change = LibraryChange( - changed_param="", current_value=proto_path, library_name=library_name - ) - diff[ChangeType.GAPIC_ADDITION].append(config_change) - - -def __convert_params_to_sorted_list(obj: Any, excluded_params=None) -> List[tuple]: - """ - Convert the parameter and its value of a given object to a sorted list of - tuples. - - Only the following types of parameters will be considered: - - - str - - bool - - list[str] - - None - - Note that built-in params, e.g., __str__, and methods will be skipped. - - :param obj: an object. - :param excluded_params: excluded params. - :return: a sorted list of tuples. - """ - if excluded_params is None: - excluded_params = set() - param_and_values = [] - for param, value in vars(obj).items(): - if ( - param in excluded_params - or param.startswith("__") # skip built-in params - or callable(getattr(obj, param)) # skip methods - # skip if the type of param is not one of the following types - # 1. str - # 2. bool - # 3. list[str] - # 4. None - or not ( - isinstance(getattr(obj, param), str) - or isinstance(getattr(obj, param), bool) - or __is_list_of_str(obj=obj, param=param) - or getattr(obj, param) is None - ) - ): - continue - param_and_values.append((param, value)) - return sorted(param_and_values) - - -def __is_list_of_str(obj: Any, param: str) -> bool: - """ - Returns True if the type of param of a given object is a list of str; False - otherwise. - - This method is a workaround of https://bugs.python.org/issue28339. - """ - value = getattr(obj, param) - if not isinstance(value, list): - return False - for v in value: - if not isinstance(v, str): - return False - return True diff --git a/sdk-platform-java/hermetic_build/common/utils/proto_path_utils.py b/sdk-platform-java/hermetic_build/common/utils/proto_path_utils.py deleted file mode 100644 index e28b4b0870ee..000000000000 --- a/sdk-platform-java/hermetic_build/common/utils/proto_path_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re - - -def find_versioned_proto_path(proto_path: str) -> str: - """ - Returns a versioned proto_path from a given proto_path; or proto_path itself - if it doesn't contain a versioned proto_path. - :param proto_path: a proto file path - :return: the versioned proto_path - """ - version_regex = re.compile(r"^v[1-9].*") - directories = proto_path.split("/") - for directory in directories: - result = version_regex.search(directory) - if result: - version = result[0] - idx = proto_path.find(version) - return proto_path[:idx] + version - return proto_path - - -def ends_with_version(proto_path: str) -> bool: - """ - Checks if a given proto_path string ends with a version identifier. - - :param proto_path: The proto_path string to check. - - :return: - True if the proto_path ends with a version, False otherwise. - """ - version_regex = re.compile(r"^v[1-9].*") - parts = proto_path.rsplit("/", 1) - if len(parts) > 1: - last_part = parts[1] - else: - last_part = parts[0] - return bool(version_regex.match(last_part)) diff --git a/sdk-platform-java/hermetic_build/library_generation/cli/entry_point.py b/sdk-platform-java/hermetic_build/library_generation/cli/entry_point.py deleted file mode 100644 index 22c4c1039a3b..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/cli/entry_point.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -from typing import Optional -import click as click -import shutil -from pathlib import Path -from library_generation.generate_repo import generate_from_yaml -from common.model.generation_config import GenerationConfig - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml that contains the - metadata about library generation. - When not set, will use generation_config.yaml from --generation-input. - When neither this or --generation-input is set default to - generation_config.yaml in the current working directory - """, -) -@click.option( - "--generation-input", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a input folder that contains - generation_config.yaml and versions.txt. - This is only used when --generation-config-path is not set. - When neither this or --generation-config-path is set default to - generation_config.yaml in the current working directory - """, -) -@click.option( - "--library-names", - required=False, - type=str, - default=None, - show_default=True, - help=""" - A list of library names that will be generated, separated by comma. - The library name of a library is the value of library_name or api_shortname, - if library_name is not specified, in the generation configuration. - - If neither --library-names or --api-path specified, all libraries in the - generation configuration will be generated. - """, -) -@click.option( - "--api-path", - required=False, - type=str, - default=None, - show_default=True, - help=""" - Path within the API root (e.g. googleapis) to the API to - generate/build/configure etc. This is expected to be a major-versioned - API directory, e.g. google/cloud/functions/v2. - - Takes precedence over --library-names when specidied. - If neither --library-names or --api-path specified, all libraries in the - generation configuration will be generated. - """, -) -@click.option( - "--repository-path", - "--output", - type=str, - default=".", - show_default=True, - help=""" - The repository path to which the generated files - will be sent. - If not specified, the repository will be generated to the current working - directory. - """, -) -@click.option( - "--api-definitions-path", - "--api-root", - type=str, - default=".", - show_default=True, - help=""" - The path to which the api definition (proto and service yaml) and its - dependencies resides. - If not specified, the path is the current working directory. - """, -) -def generate( - generation_config_path: Optional[str], - generation_input: Optional[str], - library_names: Optional[str], - api_path: Optional[str], - repository_path: str, - api_definitions_path: str, -): - """ - Generate libraries based on a generation config. - - If the `generation-config-path` is not specified the default generation - config, `$(pwd)/generation_config.yaml`, will be used. - - If `library_names` is specified, only libraries whose name can be found in - the generation config will be generated; otherwise all libraries in the - generation config will be generated. - - :raise FileNotFoundError if the specified generation config does not exist - or, in case `generation-config-path` is not specified, the default - generation config does not exist. - """ - __generate_repo_impl( - generation_config_path=generation_config_path, - generation_input=generation_input, - library_names=library_names, - api_path=api_path, - repository_path=repository_path, - api_definitions_path=api_definitions_path, - ) - - -def __generate_repo_impl( - generation_config_path: Optional[str], - generation_input: Optional[str], - library_names: Optional[str], - api_path: Optional[str], - repository_path: str, - api_definitions_path: str, -): - """ - Implementation method for generate(). - The decoupling of generate and __generate_repo_impl is - meant to allow testing of this implementation function. - """ - - # only use generation_input when generation_config_path is not provided and - # generation_input provided. generation_config_path should be deprecated after - # migration to 1pp. - default_generation_config_path = f"{os.getcwd()}/generation_config.yaml" - if generation_config_path is None and generation_input is not None: - print( - "generation_config_path is not provided, using generation-input folder provided" - ) - generation_config_path = f"{generation_input}/generation_config.yaml" - # copy versions.txt from generation_input to repository_path - # override if present. - _copy_versions_file(generation_input, repository_path) - if generation_config_path is None and generation_input is None: - print("Using default generation config path") - generation_config_path = default_generation_config_path - generation_config_path = os.path.abspath(generation_config_path) - if not os.path.isfile(generation_config_path): - raise FileNotFoundError( - f"Generation config {generation_config_path} does not exist." - ) - repository_path = os.path.abspath(repository_path) - api_definitions_path = os.path.abspath(api_definitions_path) - generation_config = GenerationConfig.from_yaml(generation_config_path) - include_library_names = _parse_library_name_from( - includes=library_names, generation_config=generation_config - ) - generate_from_yaml( - config=generation_config, - repository_path=repository_path, - api_definitions_path=api_definitions_path, - target_library_names=include_library_names, - target_api_path=api_path, - ) - - -def _copy_versions_file(generation_input_path, repository_path): - """ - Copies the versions.txt file from the generation_input folder to the repository_path. - Overrides the destination file if it already exists. - - Args: - generation_input_path (str): The path to the generation_input folder. - repository_path (str): The path to the repository folder. - """ - source_file = Path(generation_input_path) / "versions.txt" - destination_file = Path(repository_path) / "versions.txt" - - if not source_file.exists(): - destination_file.touch() - print( - f"generation-input does not contain versions.txt. " - f"Created empty versions file: {source_file}" - ) - return - try: - shutil.copy2(source_file, destination_file) - print(f"Copied '{source_file}' to '{destination_file}'") - except Exception as e: - print(f"An error occurred while copying the versions.txt: {e}") - raise # Re-raises the caught exception - - -def _needs_full_repo_generation(generation_config: GenerationConfig) -> bool: - """ - Whether you should need a full repo generation, i.e., generate all - libraries in the generation configuration. - """ - return not generation_config.is_monorepo() - - -def _parse_library_name_from( - includes: Optional[str], generation_config: GenerationConfig -) -> Optional[list[str]]: - if includes is None or _needs_full_repo_generation(generation_config): - return None - return [library_name.strip() for library_name in includes.split(",")] - - -@main.command() -@click.option( - "--generation-config-path", - required=False, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - Default to generation_config.yaml in the current working directory. - """, -) -def validate_generation_config(generation_config_path: str) -> None: - """ - Validate the given generation configuration. - """ - if generation_config_path is None: - generation_config_path = "generation_config.yaml" - try: - GenerationConfig.from_yaml(os.path.abspath(generation_config_path)) - print(f"{generation_config_path} is validated without any errors.") - except ValueError as err: - print(err) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/sdk-platform-java/hermetic_build/library_generation/cli/generate_monorepo_gapic_bom.py b/sdk-platform-java/hermetic_build/library_generation/cli/generate_monorepo_gapic_bom.py deleted file mode 100644 index b26a02c17ff9..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/cli/generate_monorepo_gapic_bom.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import click as click - -from library_generation.utils.pom_generator import generate_gapic_bom - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--repository-path", - required=True, - type=str, - help=""" - Path to which the generated pom.xml goes. - """, -) -@click.option( - "--versions-file", - required=True, - type=str, - help=""" - The file containing version of libraries. - Throw FileNotFoundError if the file doesn't exist. - """, -) -def generate(repository_path: str, versions_file: str) -> None: - generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) - - -if __name__ == "__main__": - main() diff --git a/sdk-platform-java/hermetic_build/library_generation/cli/generate_monorepo_root_pom.py b/sdk-platform-java/hermetic_build/library_generation/cli/generate_monorepo_root_pom.py deleted file mode 100644 index 8f129b63f030..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/cli/generate_monorepo_root_pom.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import click as click -from library_generation.utils.pom_generator import generate_root_pom - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--repository-path", - required=True, - type=str, - help=""" - Path to which the generated pom.xml goes. - """, -) -def generate(repository_path: str) -> None: - generate_root_pom(repository_path=repository_path) - - -if __name__ == "__main__": - main() diff --git a/sdk-platform-java/hermetic_build/library_generation/dockerignore b/sdk-platform-java/hermetic_build/library_generation/dockerignore deleted file mode 100644 index 7e978caea999..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -README.md -**/__pycache__/ -**/*.egg-info/ -**/output/ -**/build/ -**/google-cloud-java/ diff --git a/sdk-platform-java/hermetic_build/library_generation/gapic-generator-java-wrapper b/sdk-platform-java/hermetic_build/library_generation/gapic-generator-java-wrapper deleted file mode 100755 index 3f1e7bd2e7bf..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/gapic-generator-java-wrapper +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e -wrapper_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${wrapper_dir}/utils/utilities.sh" - -# Wrap gapic-generator-java.jar because protoc requires the plugin to be executable. -exec java -classpath "$(get_gapic_generator_location)" com.google.api.generator.Main diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py b/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py deleted file mode 100755 index 90253685465a..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/generate_composed_library.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This script allows generation of libraries that are composed of more than one -service version. It is achieved by calling `generate_library.sh` without -postprocessing for all service versions and then calling -postprocess_library.sh at the end, once all libraries are ready. - -Prerequisites -- Needs a folder named `output` in current working directory. This folder -is automatically detected by `generate_library.sh` and this script ensures it -contains the necessary folders and files, specifically: - - A "google" folder found in the googleapis/googleapis repository - - A "grafeas" folder found in the googleapis/googleapis repository -Note: googleapis repo is found in https://github.com/googleapis/googleapis. -""" - -import os -from pathlib import Path -from typing import List -import library_generation.utils.utilities as util -from library_generation.utils.proto_path_utils import remove_version_from -from common.model.generation_config import GenerationConfig -from common.model.gapic_config import GapicConfig -from common.model.gapic_inputs import GapicInputs -from common.model.library_config import LibraryConfig -from common.model.gapic_inputs import parse as parse_build_file -from library_generation.model.repo_config import RepoConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) - - -def generate_composed_library( - config: GenerationConfig, - library_path: str, - library: LibraryConfig, - repo_config: RepoConfig, -) -> None: - """ - Generate libraries composed of more than one service or service version - - :param config: a GenerationConfig object representing a parsed configuration - yaml - :param library_path: the path to which the generated file goes - :param library: a LibraryConfig object contained inside config, passed here - for convenience and to prevent all libraries to be processed - :param repo_config: - :return None - """ - output_folder = repo_config.output_folder - owlbot_cli_source_folder = util.sh_util("mktemp -d") - os.makedirs(f"{library_path}", exist_ok=True) - for gapic in library.get_sorted_gapic_configs(): - build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve() - print(f"build_file_folder: {build_file_folder}") - gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path) - # generate postprocessing prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml, - # owlbot.py) here because transport is parsed from BUILD.bazel, - # which lives in a versioned proto_path. The value of transport will be - # overridden by the config object if specified. Note that this override - # does not affect library generation but instead used only for - # generating postprocessing files such as README. - has_version = remove_version_from(gapic.proto_path) != gapic.proto_path - util.generate_postprocessing_prerequisite_files( - config=config, - library=library, - proto_path=util.remove_version_from(gapic.proto_path), - library_path=library_path, - transport=library.get_transport(gapic_inputs), - has_version=has_version, - ) - temp_destination_path = f"java-{gapic.proto_path.replace('/','-')}" - effective_arguments = _construct_effective_arg( - base_arguments=[], - gapic=gapic, - gapic_inputs=gapic_inputs, - temp_destination_path=temp_destination_path, - generation_config=config, - library=library, - ) - print("arguments: ") - print(effective_arguments) - print(f"Generating library from {gapic.proto_path} to {library_path}") - util.run_process_and_print_output( - ["bash", f"{script_dir}/generate_library.sh", *effective_arguments], - "Library generation", - ) - - util.sh_util( - f'build_owlbot_cli_source_folder "{library_path}"' - + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"' - + f' "{gapic.proto_path}"', - cwd=output_folder, - ) - - library_version = repo_config.get_library_version( - artifact_id=library.get_artifact_id() - ) - # call postprocess library - util.run_process_and_print_output( - [ - f"{script_dir}/postprocess_library.sh", - f"{library_path}", - "", - repo_config.versions_file, - owlbot_cli_source_folder, - str(config.is_monorepo()).lower(), - config.libraries_bom_version, - library_version, - ], - "Library postprocessing", - ) - - -def _construct_effective_arg( - base_arguments: List[str], - gapic: GapicConfig, - gapic_inputs: GapicInputs, - temp_destination_path: str, - generation_config: GenerationConfig, - library: LibraryConfig, -) -> List[str]: - """ - Construct arguments consist attributes of a GAPIC library which used in - generate_library.sh - :param base_arguments: arguments consist of tooling versions - :param gapic: an object of GapicConfig - :param gapic_inputs: an object of GapicInput - :param temp_destination_path: the path to which the generated library goes - :return: arguments containing attributes to generate a GAPIC library - """ - arguments = list(base_arguments) - arguments += util.create_argument("proto_path", gapic) - arguments += [ - "--proto_only", - gapic_inputs.proto_only, - "--gapic_additional_protos", - gapic_inputs.additional_protos, - "--transport", - gapic_inputs.transport, - "--rest_numeric_enums", - gapic_inputs.rest_numeric_enum, - "--gapic_yaml", - gapic_inputs.gapic_yaml, - "--service_config", - gapic_inputs.service_config, - "--service_yaml", - gapic_inputs.service_yaml, - "--include_samples", - gapic_inputs.include_samples, - "--repo", - util.get_library_repository(generation_config, library), - "--artifact", - library.distribution_name, - ] - arguments += ["--destination_path", temp_destination_path] - - return arguments diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_library.sh b/sdk-platform-java/hermetic_build/library_generation/generate_library.sh deleted file mode 100755 index 2af5539c32bc..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/generate_library.sh +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -# parse input parameters -while [[ $# -gt 0 ]]; do -key="$1" -case $key in - -p|--proto_path) - proto_path="$2" - shift - ;; - -d|--destination_path) - destination_path="$2" - shift - ;; - --proto_only) - proto_only="$2" - shift - ;; - --gapic_additional_protos) - gapic_additional_protos="$2" - shift - ;; - --transport) - transport="$2" - shift - ;; - --rest_numeric_enums) - rest_numeric_enums="$2" - shift - ;; - --generate_version_java) - generate_version_java="$2" - shift - ;; - - --gapic_yaml) - gapic_yaml="$2" - shift - ;; - --service_config) - service_config="$2" - shift - ;; - --service_yaml) - service_yaml="$2" - shift - ;; - --include_samples) - include_samples="$2" - shift - ;; - --os_architecture) - os_architecture="$2" - shift - ;; - --repo) - repo="$2" - shift - ;; - --artifact) - artifact="$2" - shift - ;; - *) - echo "Invalid option: [$1]" - exit 1 - ;; -esac -shift # past argument or value -done - -script_dir=$(dirname "$(readlink -f "$0")") -# source utility functions -source "${script_dir}"/utils/utilities.sh -output_folder="$(get_output_folder)" - -if [ -z "${proto_only}" ]; then - proto_only="false" -fi - -if [ -z "${generate_version_java}" ]; then - generate_version_java="true" -fi - - -if [ -z "${gapic_additional_protos}" ]; then - gapic_additional_protos="google/cloud/common_resources.proto" -fi - -if [ -z "${transport}" ]; then - transport="grpc" -fi - -if [ -z "${rest_numeric_enums}" ]; then - rest_numeric_enums="true" -fi - -if [ -z "${gapic_yaml}" ]; then - gapic_yaml="" -fi - -if [ -z "${service_config}" ]; then - service_config="" -fi - -if [ -z "${service_yaml}" ]; then - service_yaml="" -fi - -if [ -z "${include_samples}" ]; then - include_samples="true" -fi - -if [ -z "${os_architecture}" ]; then - os_architecture=$(detect_os_architecture) -fi - -if [ -z "${repo}" ]; then - repo="" -fi - -if [ -z "${artifact}" ]; then - artifact="" -fi - -temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM" -mkdir -p "${output_folder}/${destination_path}" -if [ -d "${temp_destination_path}" ]; then - # we don't want the preprocessed sources of a previous run - rm -r "${temp_destination_path}" -fi -mkdir -p "${temp_destination_path}" -##################### Section 0 ##################### -# prepare tooling -##################################################### -# the order of services entries in gapic_metadata.json is relevant to the -# order of proto file, sort the proto files with respect to their bytes to -# get a fixed order. -folder_name=$(extract_folder_name "${destination_path}") -pushd "${output_folder}" -find_depth="" -case "${proto_path}" in - "google/api" | "google/cloud" | "google/rpc") - find_depth="-maxdepth 1" - ;; -esac -proto_files=$(find "${proto_path}" ${find_depth} -type f -name "*.proto" | LC_COLLATE=C sort) -# include or exclude certain protos in grpc plugin and gapic generator java. -case "${proto_path}" in - "google/cloud") - # this proto is excluded from //google/cloud:google-apps-script-type-java - removed_proto="google/cloud/common_resources.proto" - proto_files="${proto_files//${removed_proto}/}" - ;; - "google/cloud/aiplatform/v1beta1"*) - # this proto is excluded from //google/cloud/aiplatform/v1beta1/schema:schema_proto - removed_proto="google/cloud/aiplatform/v1beta1/schema/io_format.proto" - proto_files="${proto_files//${removed_proto}/}" - ;; - "google/cloud/filestore"*) - # this proto is included in //google/cloud/filestore/v1:google-cloud-filestore-v1-java - # and //google/cloud/filestore/v1beta1:google-cloud-filestore-v1-java - proto_files="${proto_files} google/cloud/common/operation_metadata.proto" - ;; - "google/cloud/oslogin"*) - # this proto is included in //google/cloud/oslogin/v1:google-cloud-oslogin-v1-java - # and //google/cloud/oslogin/v1beta1:google-cloud-oslogin-v1-java - proto_files="${proto_files} google/cloud/oslogin/common/common.proto" - ;; - "google/cloud/visionai/v1"*) - # this proto is excluded in //google/cloud/visionai/v1:google-cloud-visionai-v1-java - # we can remove this exclusion after cl/631529749 is submitted. - removed_proto="google/cloud/visionai/v1/prediction.proto" - proto_files="${proto_files//${removed_proto}/}" - ;; -esac - -protoc_path=$(get_protoc_location) -##################### Section 1 ##################### -# generate grpc-*/ -##################################################### -if [[ ! "${transport}" == "rest" ]]; then - # do not need to generate grpc-* if the transport is `rest`. - "${protoc_path}"/protoc "--plugin=protoc-gen-rpc-plugin=$(get_grpc_plugin_location)" \ - "--rpc-plugin_out=:${temp_destination_path}/java_grpc.jar" \ - ${proto_files} # Do not quote because this variable should not be treated as one long string. - # unzip java_grpc.jar to grpc-*/src/main/java - unzip_src_files "grpc" "${temp_destination_path}" - # remove empty files in grpc-*/src/main/java - remove_empty_files "grpc" "${temp_destination_path}" - # remove grpc version in *ServiceGrpc.java file so the content is identical with bazel build. - remove_grpc_version "${temp_destination_path}" -fi -###################### Section 2 ##################### -## generate gapic-*/, part of proto-*/, samples/ -###################################################### -if [[ "${proto_only}" == "false" ]]; then - "$protoc_path"/protoc --experimental_allow_proto3_optional \ - "--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \ - "--java_gapic_out=metadata:${temp_destination_path}/java_gapic_srcjar_raw.srcjar.zip" \ - "--java_gapic_opt=$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${gapic_yaml}" "${service_config}" "${service_yaml}" "${repo}" "${artifact}" "${generate_version_java}")" \ - ${proto_files} ${gapic_additional_protos} - - unzip -o -q "${temp_destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${temp_destination_path}" - # Sync'\''d to the output file name in Writer.java. - unzip -o -q "${temp_destination_path}/temp-codegen.srcjar" -d "${temp_destination_path}/java_gapic_srcjar" - # Resource name source files. - proto_dir=${temp_destination_path}/java_gapic_srcjar/proto/src/main/java - if [ ! -d "${proto_dir}" ]; then - # Some APIs don't have resource name helpers, like BigQuery v2. - # Create an empty file so we can finish building. Gating the resource name rule definition - # on file existences go against Bazel's design patterns, so we'll simply delete all empty - # files during the final packaging process (see java_gapic_pkg.bzl) - mkdir -p "${proto_dir}" - touch "${proto_dir}"/PlaceholderFile.java - fi - # move java_gapic_srcjar/src/main to gapic-*/src. - mv_src_files "gapic" "main" "${temp_destination_path}" - # remove empty files in gapic-*/src/main/java - remove_empty_files "gapic" "${temp_destination_path}" - # move java_gapic_srcjar/src/test to gapic-*/src - mv_src_files "gapic" "test" "${temp_destination_path}" - if [ "${include_samples}" == "true" ]; then - # move java_gapic_srcjar/samples/snippets to samples/snippets - mv_src_files "samples" "main" "${temp_destination_path}" - fi -fi -##################### Section 3 ##################### -# generate proto-*/ -##################################################### -# exclude certain protos to java compiler. -case "${proto_path}" in - "google/cloud/aiplatform/v1beta1"*) - # these protos are excluded from //google/cloud/aiplatform/v1beta1:google-cloud-aiplatform-v1beta1-java - prefix="google/cloud/aiplatform/v1beta1/schema" - protos="${prefix}/annotation_payload.proto ${prefix}/annotation_spec_color.proto ${prefix}/data_item_payload.proto ${prefix}/dataset_metadata.proto ${prefix}/geometry.proto" - for removed_proto in ${protos}; do - proto_files="${proto_files//${removed_proto}/}" - done - ;; -esac -"$protoc_path"/protoc "--java_out=${temp_destination_path}/java_proto.jar" ${proto_files} -if [[ "${proto_only}" == "false" ]]; then - # move java_gapic_srcjar/proto/src/main/java (generated resource name helper class) - # to proto-*/src/main - mv_src_files "proto" "main" "${temp_destination_path}" -fi -# unzip java_proto.jar to proto-*/src/main/java -unzip_src_files "proto" "${temp_destination_path}" -# remove empty files in proto-*/src/main/java -remove_empty_files "proto" "${temp_destination_path}" -case "${proto_path}" in - "google/cloud/aiplatform/v1beta1"*) - prefix="google/cloud/aiplatform/v1beta1/schema" - protos="${prefix}/annotation_payload.proto ${prefix}/annotation_spec_color.proto ${prefix}/data_item_payload.proto ${prefix}/dataset_metadata.proto ${prefix}/geometry.proto" - for added_proto in ${protos}; do - proto_files="${proto_files} ${added_proto}" - done - ;; -esac -# copy proto files to proto-*/src/main/proto -for proto_src in ${proto_files}; do - if [[ "${proto_src}" == "google/cloud/common/operation_metadata.proto" ]]; then - continue - fi - mkdir -p "${temp_destination_path}/proto-${folder_name}/src/main/proto" - rsync -R "${proto_src}" "${temp_destination_path}/proto-${folder_name}/src/main/proto" -done -popd # output_folder -##################### Section 4 ##################### -# rm tar files -##################################################### -pushd "${temp_destination_path}" -rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar -popd # destination path - -cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}" -rm -rf "${temp_destination_path}" -exit 0 diff --git a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py b/sdk-platform-java/hermetic_build/library_generation/generate_repo.py deleted file mode 100755 index f7b31716b7a7..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/generate_repo.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import copy -import shutil -from pathlib import Path -from typing import Optional -import library_generation.utils.utilities as util -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig -from common.utils.proto_path_utils import ends_with_version -from library_generation.generate_composed_library import generate_composed_library -from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing - -from common.model.gapic_config import GapicConfig - - -def generate_from_yaml( - config: GenerationConfig, - repository_path: str, - api_definitions_path: str, - target_library_names: Optional[list[str]], - target_api_path: Optional[str] = None, -) -> None: - """ - Based on the generation config, generates libraries via - generate_composed_library.py - - :param config: a GenerationConfig object. - :param repository_path: The repository path to which the generated files - will be sent. - :param api_definitions_path: The path to where the api definition resides. - :param target_library_names: a list of libraries to be generated. - If specified, only the library whose library_name is in target_library_names - will be generated. - If specified with an empty list, then no library will be generated. - If not specified, all libraries in the configuration yaml will be generated. - """ - target_libraries = get_target_libraries( - config=config, - target_library_names=target_library_names, - target_api_path=target_api_path, - ) - repo_config = util.prepare_repo( - gen_config=config, library_config=target_libraries, repo_path=repository_path - ) - # copy api definition to output folder. - shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) - for library_path, library in repo_config.get_libraries().items(): - print(f"generating library {library.get_library_name()}") - generate_composed_library( - config=config, - library_path=library_path, - library=library, - repo_config=repo_config, - ) - - if not config.is_monorepo(): - return - - monorepo_postprocessing( - repository_path=repository_path, versions_file=repo_config.versions_file - ) - - # cleanup temp output folder - try: - shutil.rmtree(Path(repo_config.output_folder)) - print(f"Directory {repo_config.output_folder} and its contents removed.") - except OSError as e: - print(f"Error: {e} - Failed to remove directory {repo_config.output_folder}.") - raise - - -def get_target_libraries( - config: GenerationConfig, - target_library_names: list[str] = None, - target_api_path: Optional[str] = None, -) -> list[LibraryConfig]: - """ - Returns LibraryConfig objects whose library_name is in target_library_names. - - :param config: a GenerationConfig object. - :param target_library_names: library_name of target libraries. - If not specified, all libraries in the given config will be returned. - :return: LibraryConfig objects. - """ - if target_api_path is None: - return _get_target_libraries_from_name(config, target_library_names) - return _get_target_libraries_from_api_path(config, target_api_path) - - -def _get_target_libraries_from_name( - config: GenerationConfig, target_library_names: list[str] = None -) -> list[LibraryConfig]: - """ - Returns LibraryConfig objects whose library_name is in target_library_names. - - :param config: a GenerationConfig object. - :param target_library_names: library_name of target libraries. - If not specified, all libraries in the given config will be returned. - :return: LibraryConfig objects. - """ - if target_library_names is None: - return config.libraries - target_libraries = set(target_library_names) - return [ - library - for library in config.libraries - if library.get_library_name() in target_libraries - ] - - -def _get_target_libraries_from_api_path( - config: GenerationConfig, target_api_path: Optional[str] = None -) -> list[LibraryConfig]: - """ - Retrieves a copy of the LibraryConfig objects that contain the specified - target API path, without other proto_path versions that were not specified. - dependency proto_paths are kept. - - :param config: The GenerationConfig object. - :param target_api_path: The target proto path to search for. - - :return: - A list of LibraryConfig objects matching the target API path, or an - empty list if no matches are found. - """ - if not ends_with_version(target_api_path): - raise ValueError("api_path is not ending with a version is not supported") - target_libraries = [] - for library in config.libraries: - target_library = copy.deepcopy(library) - gapic_config_list = [] - for item in library.gapic_configs: - if item.proto_path == target_api_path or not ends_with_version( - item.proto_path - ): - gapic_config_list.append(GapicConfig(item.proto_path)) - if gapic_config_list: - target_library.set_gapic_configs(gapic_config_list) - target_libraries.append(target_library) - return target_libraries - return [] diff --git a/sdk-platform-java/hermetic_build/library_generation/model/bom_config.py b/sdk-platform-java/hermetic_build/library_generation/model/bom_config.py deleted file mode 100644 index b562407eb746..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/model/bom_config.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class BomConfig: - """ - Class that represents an entry in dependencyManagement section. - """ - - def __init__( - self, - group_id: str, - artifact_id: str, - version: str, - version_annotation: str, - is_import: bool = True, - ): - self.group_id = group_id - self.artifact_id = artifact_id - self.version = version - self.version_annotation = version_annotation - self.is_import = is_import - - def __lt__(self, another): - return self.group_id < another.group_id or ( - self.group_id == another.group_id and self.artifact_id < another.artifact_id - ) - - def __eq__(self, another): - return ( - self.group_id == another.group_id - and self.artifact_id == another.artifact_id - ) diff --git a/sdk-platform-java/hermetic_build/library_generation/model/repo_config.py b/sdk-platform-java/hermetic_build/library_generation/model/repo_config.py deleted file mode 100644 index e9af0a982040..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/model/repo_config.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from common.model.library_config import LibraryConfig - -GRPC_PREFIX = "grpc-" -PROTO_PREFIX = "proto-" -NEW_CLIENT_VERSION = "0.0.0" - - -class RepoConfig: - """ - Class that represents a generated repository - """ - - def __init__( - self, - output_folder: str, - libraries: dict[str, LibraryConfig], - versions_file: str, - ): - """ - Init a RepoConfig object - - :param output_folder: the path to which the generated repo goes - :param libraries: a mapping from library_path to LibraryConfig object - :param versions_file: the path of versions.txt used in post-processing - """ - self.output_folder = output_folder - self.libraries = libraries - self.versions_file = versions_file - self.library_versions = self.__parse_versions() - - def get_libraries(self) -> dict[str, LibraryConfig]: - return self.libraries - - def get_library_version(self, artifact_id: str) -> str: - """ - Returns the version of a given artifact ID. - If the artifact ID is not managed, i.e., a new client, returns `0.0.0`. - - :param artifact_id: the Maven artifact ID. - :return: the version of the artifact. - """ - return self.library_versions.get(artifact_id, NEW_CLIENT_VERSION) - - def __parse_versions(self) -> dict[str, str]: - """ - For a given versions.txt file (defined in self.versions_file) - creates a map of artifact-id to its version - - :return: a map "artifact-id -> version" - """ - library_versions = dict() - with open(self.versions_file) as f: - for line in f.readlines(): - sections = line.split(":") - # skip comments and empty lines. - if len(sections) != 3: - continue - artifact_id = sections[0] - released_version = sections[1] - if artifact_id.startswith(GRPC_PREFIX) or artifact_id.startswith( - PROTO_PREFIX - ): - continue - library_versions[artifact_id] = released_version - return library_versions diff --git a/sdk-platform-java/hermetic_build/library_generation/postprocess_library.sh b/sdk-platform-java/hermetic_build/library_generation/postprocess_library.sh deleted file mode 100755 index 6e92e1ddb295..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/postprocess_library.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash -# -# Main functions to interact with owlbot post-processor - -# Runs the java owlbot post-processor. The resulting post-processed -# library gets stored in the $postprocessing_target argument -# Arguments -# 1 - postprocessing_target: path where the postprocessor will run. This folder -# has the following requirements -# - a .repo-metadata.json file must be present -# - an owlbot.py file must be present -# - an .OwlBot-hermetic.yaml file must be present -# 2 - preprocessed_sources_path: used to transfer the raw grpc, proto and gapic -# libraries into the postprocessing_target via copy-code -# 3 - versions_file: path to file containing versions to be applied to the poms -# 4 - owlbot_cli_source_folder: alternative folder with a structure exactly like -# googleapis-gen. It will be used instead of preprocessed_sources_path if -# 5 - is_monorepo: whether this library is a monorepo, which implies slightly -# different logic -# 6 - libraries_bom_version: used by our implementation of owlbot to render the -# readme -# 7 - library_version: used by our implementation of owlbot to render the -# readme -set -exo pipefail -scripts_root=$(dirname "$(readlink -f "$0")") - -postprocessing_target=$1 -preprocessed_sources_path=$2 -versions_file=$3 -owlbot_cli_source_folder=$4 -is_monorepo=$5 -libraries_bom_version=$6 -library_version=$7 -owlbot_yaml_file_name=".OwlBot-hermetic.yaml" - -source "${scripts_root}"/utils/utilities.sh - -declare -a required_inputs=("postprocessing_target" "versions_file" "is_monorepo") -for required_input in "${required_inputs[@]}"; do - if [[ -z "${!required_input}" ]]; then - echo "missing required ${required_input} argument, please specify one" - exit 1 - fi -done - -for owlbot_file in ".repo-metadata.json" "owlbot.py" "${owlbot_yaml_file_name}" -do - if [[ $(find "${postprocessing_target}" -name "${owlbot_file}" | wc -l) -eq 0 ]]; then - echo "necessary file for postprocessing '${owlbot_file}' was not found in postprocessing_target" - echo "please provide a postprocessing_target folder that is compatible with the OwlBot Java postprocessor" - exit 1 - fi -done - -if [[ -z "${owlbot_cli_source_folder}" ]]; then - owlbot_cli_source_folder=$(mktemp -d) - build_owlbot_cli_source_folder "${postprocessing_target}" "${owlbot_cli_source_folder}" "${preprocessed_sources_path}" -fi - - -# we determine the location of the .OwlBot-hermetic.yaml file by checking if the target -# folder is a monorepo folder or not -if [[ "${is_monorepo}" == "true" ]]; then - # the deep-remove-regex and deep-preserve-regex of the .OwlBot-hermetic.yaml - # files in the monorepo libraries assume that `copy-code` is run - # from the root of the monorepo. However, we call `copy-code` from inside each - # library, so a path like `/java-asset/google-.*/src` will not have - # any effect. We solve this by creating a temporary owlbot yaml with - # the patched paths. - # For example, we convert - # - "/java-asset/google-.*/src" - # to - # - "/google-.*/src" - - library_name=$(basename "${postprocessing_target}") - normalize_owlbot_yaml "${postprocessing_target}/${owlbot_yaml_file_name}" "${postprocessing_target}/.OwlBot.hermetic.yaml" "${library_name}" - owlbot_yaml_relative_path=".OwlBot.hermetic.yaml" -else - owlbot_yaml_relative_path=".github/${owlbot_yaml_file_name}" -fi - -# Default values for running copy-code directly from host -repo_workspace="/workspace" -preprocessed_libraries_binding="${owlbot_cli_source_folder}" - -pushd "${postprocessing_target}" - -# Backs up all files matching `stub/Version.java` -# This is a workaround to prevent owl-bot-copy from omitting -# Version.java files when transferring the generated code -backup_dir=$(mktemp -d) -backup_version_java "${PWD}" "${backup_dir}" - -owl-bot copy-code \ - --source-repo-commit-hash=none \ - --source-repo="${owlbot_cli_source_folder}" \ - --config-file="${owlbot_yaml_relative_path}" - - -# clean the custom owlbot yaml -if [[ "${is_monorepo}" == "true" ]]; then - rm "${postprocessing_target}/.OwlBot.hermetic.yaml" -fi - -# run the postprocessor -echo 'running owl-bot post-processor' -pushd "${postprocessing_target}" -bash "${scripts_root}/owlbot/bin/entrypoint.sh" \ - "${scripts_root}" \ - "${versions_file}" \ - "${is_monorepo}" \ - "${libraries_bom_version}" \ - "${library_version}" - -restore_version_java "${PWD}" "${backup_dir}" -rm -rf "${backup_dir}" - -popd # postprocessing_target - diff --git a/sdk-platform-java/hermetic_build/library_generation/setup.py b/sdk-platform-java/hermetic_build/library_generation/setup.py index cce0a27d8b1f..f6a75556fe85 100755 --- a/sdk-platform-java/hermetic_build/library_generation/setup.py +++ b/sdk-platform-java/hermetic_build/library_generation/setup.py @@ -14,11 +14,6 @@ }, package_data={ "library_generation": [ - "generate_library.sh", - "postprocess_library.sh", - "utils/utilities.sh", - "templates/*.j2", - "gapic-generator-java-wrapper", "owlbot/bin/*.sh", "owlbot/src/*.py", "owlbot/src/poms/*.py", diff --git a/sdk-platform-java/hermetic_build/library_generation/templates/gapic-libraries-bom.xml.j2 b/sdk-platform-java/hermetic_build/library_generation/templates/gapic-libraries-bom.xml.j2 deleted file mode 100644 index 45dbdf42ce3c..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/templates/gapic-libraries-bom.xml.j2 +++ /dev/null @@ -1,37 +0,0 @@ - - - 4.0.0 - com.google.cloud - gapic-libraries-bom - pom - {{ monorepo_version }} - Google Cloud Java BOM - - BOM for the libraries in google-cloud-java repository. Users should not - depend on this artifact explicitly because this BOM is an implementation - detail of the Libraries BOM. - - - - google-cloud-pom-parent - com.google.cloud - {{ monorepo_version }} - ../google-cloud-pom-parent/pom.xml - - - - - {%- for bom_config in bom_configs %} - - {{ bom_config.group_id }} - {{ bom_config.artifact_id }} - {{ bom_config.version }} - {%- if bom_config.is_import %} - pom - import - {%- endif %} - - {%- endfor %} - - - diff --git a/sdk-platform-java/hermetic_build/library_generation/templates/owlbot.py.j2 b/sdk-platform-java/hermetic_build/library_generation/templates/owlbot.py.j2 deleted file mode 100644 index b0d41ff0498c..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/templates/owlbot.py.j2 +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import synthtool as s -{% if should_include_templates %}from synthtool.languages import java{% endif %} - -for library in s.get_staging_dirs(): - # put any special-case replacements here - s.move(library) - -s.remove_staging_dirs(){% if should_include_templates %} -java.common_templates( - monorepo=True,{% if template_excludes %} - excludes=[ - {%- for exclude in template_excludes %} - "{{ exclude }}"{% if not loop.last %},{% endif %} - {%- endfor %} -]) -{%- else %} -) -{%- endif %} -{%- endif %} \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/templates/owlbot.yaml.monorepo.j2 b/sdk-platform-java/hermetic_build/library_generation/templates/owlbot.yaml.monorepo.j2 deleted file mode 100644 index d3f29de32a7e..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/templates/owlbot.yaml.monorepo.j2 +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{% if artifact_id %} -deep-remove-regex: -- "/{{ module_name }}/grpc-google-.*/src" -- "/{{ module_name }}/proto-google-.*/src" -- "/{{ module_name }}/google-.*/src" -- "/{{ module_name }}/samples/snippets/generated" - -deep-preserve-regex: -- "/{{ module_name }}/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" - -deep-copy-regex: -{%- if has_version %} -- source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_id }}-$1/src" -- source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_id }}-$1/src" -- source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_id }}/src" -- source: "/{{ proto_path }}/(v.*)/.*-java/samples/snippets/generated" - dest: "/owl-bot-staging/{{ module_name }}/$1/samples/snippets/generated" -{%- else %} -- source: "/{{ proto_path }}/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/proto-{{ artifact_id }}/src" -- source: "/{{ proto_path }}/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/grpc-{{ artifact_id }}/src" -- source: "/{{ proto_path }}/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/{{ artifact_id }}/src" -- source: "/{{ proto_path }}/.*-java/samples/snippets/generated" - dest: "/owl-bot-staging/{{ module_name }}/samples/snippets/generated" -{%- endif %} -{%- endif %} - -api-name: {{ api_shortname }} diff --git a/sdk-platform-java/hermetic_build/library_generation/templates/root-pom.xml.j2 b/sdk-platform-java/hermetic_build/library_generation/templates/root-pom.xml.j2 deleted file mode 100644 index 56704aa4e483..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/templates/root-pom.xml.j2 +++ /dev/null @@ -1,88 +0,0 @@ - - - 4.0.0 - google-cloud-java - com.google.cloud - 0.201.0 - pom - - - true - - - - gapic-libraries-bom - google-cloud-jar-parent - google-cloud-pom-parent - {%- for module in modules %} - {{ module }} - {%- endfor %} - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.1.3 - - true - - - - - - - - release-staging-repository - - - - !gpg.executable - - - - - sonatype-nexus-snapshots - https://google.oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-staging - https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.7.0 - true - - sonatype-nexus-staging - https://google.oss.sonatype.org/ - false - - - - - - - release-non-google-oss-sonatype - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - ossrh - https://oss.sonatype.org/ - - - - - - - diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/cli/__init__.py b/sdk-platform-java/hermetic_build/library_generation/tests/cli/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py deleted file mode 100644 index 2e8890a78e43..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py +++ /dev/null @@ -1,264 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import shutil -import unittest -from pathlib import Path -from unittest.mock import patch, ANY -from click.testing import CliRunner -from library_generation.cli.entry_point import ( - generate, - validate_generation_config, - __generate_repo_impl as generate_impl, -) -from common.model.generation_config import GenerationConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) -test_resource_dir = os.path.join(script_dir, "..", "resources", "test-config") - - -class EntryPointTest(unittest.TestCase): - def test_entry_point_without_default_config_raise_file_exception(self): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke(generate) - self.assertEqual(1, result.exit_code) - self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex( - result.exception.args[0], "generation_config.yaml does not exist." - ) - - def test_entry_point_with_invalid_config_raise_file_exception(self): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, ["--generation-config-path=/non-existent/file"] - ) - self.assertEqual(1, result.exit_code) - self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex(result.exception.args[0], "/non-existent/file does not exist.") - - def test_entry_point_with_invalid_generation_input_raise_file_exception( - self, - ): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke(generate, ["--generation-input=/non-existent/folder"]) - self.assertEqual(1, result.exit_code) - self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex( - result.exception.args[0], - "/non-existent/folder/generation_config.yaml does not exist.", - ) - - def test_validate_generation_config_succeeds( - self, - ): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - validate_generation_config, - [f"--generation-config-path={test_resource_dir}/generation_config.yaml"], - ) - self.assertEqual(0, result.exit_code) - - def test_validate_generation_config_with_duplicate_library_name_raise_file_exception( - self, - ): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - validate_generation_config, - [ - f"--generation-config-path={test_resource_dir}/generation_config_with_duplicate_library_name.yaml" - ], - ) - self.assertEqual(1, result.exit_code) - self.assertEqual(SystemExit, result.exc_info[0]) - self.assertRegex( - result.output, - "have the same library name", - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_without_library_names_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). generate() should call generate_from_yaml() - with target_library_names=None in order to trigger the full generation - """ - config_path = f"{test_resource_dir}/generation_config.yaml" - self.assertFalse(GenerationConfig.from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - generation_config_path=config_path, - generation_input=None, - library_names=None, - api_path=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=None, - target_api_path=None, - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_with_library_names_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). - generate() should call generate_from_yaml() with - target_library_names equals includes. - """ - config_path = f"{test_resource_dir}/generation_config.yaml" - self.assertFalse(GenerationConfig.from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - generation_config_path=config_path, - generation_input=None, - library_names="non-existent-library", - api_path=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=None, - target_api_path=None, - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_library_names_trigger_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - generate() should call generate_from_yaml() with - target_library_names=changed libraries which does not trigger the full - generation. - """ - config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(GenerationConfig.from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - generation_config_path=config_path, - generation_input=None, - library_names=None, - api_path=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=None, - target_api_path=None, - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_library_names_trigger_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - generate() should call generate_from_yaml() with - target_library_names=changed libraries which does not trigger the full - generation. - """ - config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(GenerationConfig.from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - generation_config_path=config_path, - generation_input=None, - library_names="asset", - api_path=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["asset"], - target_api_path=None, - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("common.model.generation_config.GenerationConfig.from_yaml") - def test_generate_provide_generation_input( - self, - mock_from_yaml, - mock_generate_from_yaml, - ): - """ - This test confirms that when no generation_config_path and - only generation_input is provided, it looks inside this path - for generation config and creates versions file when not exists - """ - config_path = f"{test_resource_dir}/generation_config.yaml" - self._create_folder_in_current_dir("test-output") - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - generation_config_path=None, - generation_input=test_resource_dir, - library_names="asset", - api_path=None, - repository_path="./test-output", - api_definitions_path=".", - ) - mock_from_yaml.assert_called_with(os.path.abspath(config_path)) - self.assertTrue(os.path.exists(f"test-output/versions.txt")) - - def tearDown(self): - # clean up after - if os.path.exists("./output"): - shutil.rmtree(Path("./output")) - if os.path.exists("./test-output"): - shutil.rmtree(Path("./test-output")) - - def _create_folder_in_current_dir(self, folder_name): - """Creates a folder in the current directory.""" - try: - os.makedirs( - folder_name, exist_ok=True - ) # exist_ok prevents errors if folder exists - print(f"Folder '{folder_name}' created successfully.") - except OSError as e: - print(f"Error creating folder '{folder_name}': {e}") diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/cli/versions.txt b/sdk-platform-java/hermetic_build/library_generation/tests/cli/versions.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/generate_library_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/generate_library_unit_tests.py deleted file mode 100644 index de3b73d0e361..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/generate_library_unit_tests.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import subprocess -import unittest -import os -from library_generation.utils.utilities import ( - run_process_and_print_output as bash_call, - run_process_and_get_output_string as get_bash_call_output, -) -from library_generation.generate_composed_library import _construct_effective_arg -from common.model.gapic_config import GapicConfig -from common.model.gapic_inputs import GapicInputs -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) - - -class GenerateLibraryUnitTests(unittest.TestCase): - """ - Confirms the correct behavior of `library_generation/utils/utilities.sh`. - - Note that there is an already existing, shell-based, test suite for - generate_library.sh, but these tests will soon be transferred to this one as - an effort to unify the implementation of the Hermetic Build scripts as - python-only. New tests for `utilities.sh` should be added in this file. - """ - - TEST_ARCHITECTURE = "linux-x86_64" - - def setUp(self): - # we create a simulated home folder that has a fake generator jar - # in its well-known location - self.simulated_home = get_bash_call_output("mktemp -d") - bash_call(f"mkdir {self.simulated_home}/.library_generation") - # We create a per-test directory where all output files will be created into. - # Each folder will be deleted after its corresponding test finishes. - test_dir = get_bash_call_output("mktemp -d") - self.output_folder = self._run_command_and_get_sdout( - "get_output_folder", - cwd=test_dir, - ) - bash_call(f"mkdir {self.output_folder}") - - def tearDown(self): - bash_call(f"rm -rf {self.simulated_home}") - - def _run_command(self, command, **kwargs): - env = os.environ.copy() - env["HOME"] = self.simulated_home - if "cwd" not in kwargs: - kwargs["cwd"] = self.output_folder - return bash_call( - [ - "bash", - "-exc", - f"source {script_dir}/../utils/utilities.sh " + f"&& {command}", - ], - exit_on_fail=False, - env=env, - **kwargs, - ) - - def _run_command_and_get_sdout(self, command, **kwargs): - return self._run_command( - command, stderr=subprocess.PIPE, **kwargs - ).stdout.decode()[:-1] - - def test_get_generator_location_with_env_returns_env(self): - os.environ["GAPIC_GENERATOR_LOCATION"] = "/gapic-generator-java" - result = self._run_command_and_get_sdout("get_gapic_generator_location") - self.assertEqual("/gapic-generator-java", result) - os.environ.pop("GAPIC_GENERATOR_LOCATION") - - def test_get_generator_location_without_env_with_local_returns_local(self): - bash_call( - f"touch {self.simulated_home}/.library_generation/gapic-generator-java.jar" - ) - result = self._run_command_and_get_sdout("get_gapic_generator_location") - self.assertEqual( - f"{self.simulated_home}/.library_generation/gapic-generator-java.jar", - result, - ) - - def test_get_generator_location_with_no_env_no_local_file_failed(self): - result = self._run_command("get_gapic_generator_location") - self.assertEqual(1, result.returncode) - self.assertRegex(result.stdout.decode(), "Can't find GAPIC generator in") - - def test_get_protoc_location_with_env_returns_env(self): - os.environ["DOCKER_PROTOC_LOCATION"] = "/protoc" - result = self._run_command_and_get_sdout("get_protoc_location") - self.assertEqual("/protoc", result) - os.environ.pop("DOCKER_PROTOC_LOCATION") - - def test_get_protoc_location_without_env_with_local_returns_local(self): - bash_call(f"mkdir -p {self.simulated_home}/.library_generation/bin") - result = self._run_command_and_get_sdout("get_protoc_location") - self.assertEqual( - f"{self.simulated_home}/.library_generation/bin", - result, - ) - - def test_get_protoc_location_with_no_env_no_local_file_failed(self): - result = self._run_command("get_protoc_location") - self.assertEqual(1, result.returncode) - self.assertRegex(result.stdout.decode(), "Can't find protoc in") - - def test_get_grpc_plugin_location_with_env_returns_env(self): - os.environ["DOCKER_GRPC_LOCATION"] = "/grpc" - result = self._run_command_and_get_sdout("get_grpc_plugin_location") - self.assertEqual("/grpc", result) - os.environ.pop("DOCKER_GRPC_LOCATION") - - def test_get_grpc_plugin_location_without_env_with_local_returns_local(self): - bash_call( - f"touch {self.simulated_home}/.library_generation/protoc-gen-grpc-java.exe" - ) - result = self._run_command_and_get_sdout("get_grpc_plugin_location") - self.assertEqual( - f"{self.simulated_home}/.library_generation/protoc-gen-grpc-java.exe", - result, - ) - - def test_get_grpc_plugin_location_with_no_env_no_local_file_failed(self): - result = self._run_command("get_grpc_plugin_location") - self.assertEqual(1, result.returncode) - self.assertRegex(result.stdout.decode(), "Can't find grpc plugin in") - - def test_get_formatter_location_with_env_returns_env(self): - os.environ["JAVA_FORMATTER_LOCATION"] = "/java-formatter" - result = self._run_command_and_get_sdout("get_java_formatter_location") - self.assertEqual("/java-formatter", result) - os.environ.pop("JAVA_FORMATTER_LOCATION") - - def test_get_formatter_location_without_env_with_local_returns_local(self): - bash_call( - f"touch {self.simulated_home}/.library_generation/google-java-format.jar" - ) - result = self._run_command_and_get_sdout("get_java_formatter_location") - self.assertEqual( - f"{self.simulated_home}/.library_generation/google-java-format.jar", - result, - ) - - def test_get_formatter_location_with_no_env_no_local_file_failed(self): - result = self._run_command("get_java_formatter_location") - self.assertEqual(1, result.returncode) - self.assertRegex(result.stdout.decode(), "Can't find Java formatter in") - - def test_construct_effective_arg_confirm_repo_and_artifact(self): - gapic_config = GapicConfig(proto_path="google/cloud/test/v1") - gapic_inputs = GapicInputs( - proto_only="false", - additional_protos="google/cloud/common_resources.proto", - transport="grpc", - rest_numeric_enum="true", - gapic_yaml="google/cloud/test/v1/test_gapic.yaml", - service_config="google/cloud/test/v1/test_service_config.json", - service_yaml="google/cloud/test/v1/test.yaml", - include_samples="true", - ) - library_config = LibraryConfig( - api_shortname="test", - name_pretty="Test API", - product_documentation="https://cloud.google.com/test", - api_description="Test API description", - gapic_configs=[gapic_config], - library_name="test-library", - distribution_name="com.google.cloud:google-cloud-test-library", - ) - generation_config = GenerationConfig( - gapic_generator_version="2.34.0", - googleapis_commitish="main", - libraries=[library_config], - ) - - effective_args = _construct_effective_arg( - base_arguments=[], - gapic=gapic_config, - gapic_inputs=gapic_inputs, - temp_destination_path="temp", - generation_config=generation_config, - library=library_config, - ) - - self.assertIn("--repo", effective_args) - self.assertIn("googleapis/java-test-library", effective_args) - self.assertIn("--artifact", effective_args) - self.assertIn("com.google.cloud:google-cloud-test-library", effective_args) diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/generate_library_unit_tests.sh b/sdk-platform-java/hermetic_build/library_generation/tests/generate_library_unit_tests.sh deleted file mode 100755 index 95bba22eb3cd..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/generate_library_unit_tests.sh +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env bash - -set -xeo pipefail - -# Unit tests against ../utilities.sh -script_dir=$(dirname "$(readlink -f "$0")") -source "${script_dir}"/test_utilities.sh - -# we simulate a properly prepared environment (i.e. generator jar in its -# well-known location). Tests confirming the opposite case will make sure this -# environment is restored -readonly SIMULATED_HOME=$(mktemp -d) -mkdir "${SIMULATED_HOME}/.library_generation" -touch "${SIMULATED_HOME}/.library_generation/gapic-generator-java.jar" -touch "${SIMULATED_HOME}/.library_generation/google-java-format.jar" -HOME="${SIMULATED_HOME}" source "${script_dir}"/../utils/utilities.sh - -# Unit tests -extract_folder_name_test() { - local path="google/cloud/aiplatform/v1/google-cloud-aiplatform-v1-java" - local folder_name - folder_name=$(extract_folder_name "${path}") - assertEquals "google-cloud-aiplatform-v1-java" "${folder_name}" -} - -get_gapic_opts_with_rest_test() { - local proto_path="${script_dir}/resources/gapic_options" - local transport="grpc" - local rest_numeric_enums="true" - local repo="googleapis/google-cloud-java" - local artifact="google-cloud-library" - local gapic_opts - gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "" "${repo}" "${artifact}")" - assertEquals \ - "transport=grpc,rest-numeric-enums,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml,repo=${repo},artifact=${artifact}" \ - "${gapic_opts}" -} - -get_gapic_opts_without_rest_test() { - local proto_path="${script_dir}/resources/gapic_options" - local transport="grpc" - local rest_numeric_enums="false" - local repo="googleapis/google-cloud-java" - local artifact="google-cloud-library" - local gapic_opts - gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "" "${repo}" "${artifact}")" - assertEquals \ - "transport=grpc,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml,repo=${repo},artifact=${artifact}" \ - "$gapic_opts" -} - -get_gapic_opts_with_non_default_test() { - local proto_path="${script_dir}/resources/gapic_options" - local transport="grpc" - local rest_numeric_enums="false" - local repo="googleapis/google-cloud-java" - local artifact="google-cloud-library" - local gapic_opts - gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${proto_path}/example_gapic.yaml" "${proto_path}/example_grpc_service_config.json" "${proto_path}/example.yaml" "${repo}" "${artifact}")" - assertEquals \ - "transport=grpc,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml,repo=${repo},artifact=${artifact}" \ - "$gapic_opts" -} - -get_gapic_opts_with_generate_version_java_true_test() { - local proto_path="${script_dir}/resources/gapic_options" - local transport="grpc" - local rest_numeric_enums="false" - local repo="googleapis/google-cloud-java" - local artifact="google-cloud-library" - local generate_version_java="true" - local gapic_opts - gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "" "${repo}" "${artifact}" "${generate_version_java}")" - assertEquals \ - "transport=grpc,generate-version-java,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml,repo=${repo},artifact=${artifact}" \ - "$gapic_opts" -} - -remove_grpc_version_test() { - local destination_path="${script_dir}/resources/gapic_options" - cp "${destination_path}/QueryServiceGrpc_copy.java" "${destination_path}/QueryServiceGrpc.java" - remove_grpc_version "${destination_path}" - local res=0 - if ! grep -q 'value = "by gRPC proto compiler",' "${destination_path}/QueryServiceGrpc.java"; then - echo "Error: grpc version is not removed." - res=1 - fi - - assertEquals 0 $((res)) - rm "${destination_path}/QueryServiceGrpc.java" -} - -download_protoc_succeed_with_valid_version_linux_test() { - download_protoc "23.2" "linux-x86_64" - assertFileOrDirectoryExists "bin" - rm -rf "bin" -} - -download_protoc_succeed_with_valid_version_macos_test() { - download_protoc "23.2" "osx-x86_64" - assertFileOrDirectoryExists "bin" - rm -rf "bin" -} - -download_protoc_failed_with_invalid_version_linux_test() { - local res=0 - $(download_protoc "22.99" "linux-x86_64") || res=$? - assertEquals 1 $((res)) -} - -download_protoc_failed_with_invalid_arch_test() { - local res=0 - $(download_protoc "23.2" "customized-x86_64") || res=$? - assertEquals 1 $((res)) -} - -download_grpc_plugin_succeed_with_valid_version_linux_test() { - download_grpc_plugin "1.55.1" "linux-x86_64" - assertFileOrDirectoryExists "protoc-gen-grpc-java.exe" - rm "protoc-gen-grpc-java.exe" -} - -download_grpc_plugin_succeed_with_valid_version_macos_test() { - download_grpc_plugin "1.55.1" "osx-x86_64" - assertFileOrDirectoryExists "protoc-gen-grpc-java.exe" - rm "protoc-gen-grpc-java.exe" -} - -download_grpc_plugin_failed_with_invalid_version_linux_test() { - local res=0 - $(download_grpc_plugin "0.99.0" "linux-x86_64") || res=$? - assertEquals 1 $((res)) -} - -download_grpc_plugin_failed_with_invalid_arch_test() { - local res=0 - $(download_grpc_plugin "1.55.1" "customized-x86_64") || res=$? - assertEquals 1 $((res)) -} - -generate_library_failed_with_invalid_generator_version() { - local destination="google-cloud-alloydb-v1-java" - local res=0 - cd "${script_dir}/resources" - bash "${script_dir}"/../generate_library.sh \ - -p google/cloud/alloydb/v1 \ - -d ../"${destination}" \ - --protoc_version 23.2 \ - --grpc_version 1.55.1 \ - --transport grpc+rest \ - --rest_numeric_enums true || res=$? - assertEquals 1 $((res)) - # still need to clean up potential downloaded tooling. - cleanup "${destination}" -} - -generate_library_failed_with_invalid_protoc_version() { - local destination="google-cloud-alloydb-v1-java" - local res=0 - cd "${script_dir}/resources" - bash "${script_dir}"/../generate_library.sh \ - -p google/cloud/alloydb/v1 \ - -d ../"${destination}" \ - --protoc_version 22.99 \ - --grpc_version 1.55.1 \ - --transport grpc+rest \ - --rest_numeric_enums true || res=$? - assertEquals 1 $((res)) - # still need to clean up potential downloaded tooling. - cleanup "${destination}" -} - -generate_library_failed_with_invalid_grpc_version() { - local destination="google-cloud-alloydb-v1-java" - local res=0 - cd "${script_dir}/resources" - bash "${script_dir}"/../generate_library.sh \ - -p google/cloud/alloydb/v1 \ - -d ../output/"${destination}" \ - --grpc_version 0.99.0 \ - --transport grpc+rest \ - --rest_numeric_enums true || res=$? - assertEquals 1 $((res)) - # still need to clean up potential downloaded tooling. - cleanup "${destination}" -} - -copy_directory_if_exists_valid_folder_succeeds() { - local source_folder="${script_dir}/resources" - local destination="${script_dir}/test_destination_folder" - mkdir -p "${destination}" - copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" - n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) - rm -rf "${destination}" - assertEquals 1 ${n_matching_folders} -} - -copy_directory_if_exists_invalid_folder_does_not_copy() { - local source_folder="${script_dir}/non-existent" - local destination="${script_dir}/test_destination_folder" - mkdir -p "${destination}" - copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" - n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) || res=$? - rm -rf "${destination}" - assertEquals 0 ${n_matching_folders} -} - -get_proto_path_from_preprocessed_sources_valid_library_succeeds() { - local sources="${script_dir}/resources/proto_path_library" - local proto_path=$(get_proto_path_from_preprocessed_sources "${sources}") - assertEquals "google/cloud/test/v1" ${proto_path} -} - -get_proto_path_from_preprocessed_sources_empty_library_fails() { - local sources=$(mktemp -d) - ( - get_proto_path_from_preprocessed_sources "${sources}" - ) || res=$? - assertEquals 1 ${res} -} - -get_proto_path_from_preprocessed_sources_multiple_proto_dirs_fails() { - local sources="${script_dir}/resources/proto_path_library_multiple_protos" - ( - get_proto_path_from_preprocessed_sources "${sources}" - ) || res=$? - assertEquals 1 ${res} -} - -backup_and_restore_version_java_succeeds() { - local target_dir=$(mktemp -d) - local backup_dir=$(mktemp -d) - - mkdir -p "${target_dir}/google-cloud-test/src/main/java/com/google/cloud/test/v1/stub" - echo "version1" > "${target_dir}/google-cloud-test/src/main/java/com/google/cloud/test/v1/stub/Version.java" - - backup_version_java "${target_dir}" "${backup_dir}" - - # Simulate owl-bot overwriting the file - echo "version2" > "${target_dir}/google-cloud-test/src/main/java/com/google/cloud/test/v1/stub/Version.java" - - restore_version_java "${target_dir}" "${backup_dir}" - - # The original "version1" should have been restored - local restored_content=$(cat "${target_dir}/google-cloud-test/src/main/java/com/google/cloud/test/v1/stub/Version.java") - assertEquals "version1" "${restored_content}" - - rm -rf "${target_dir}" "${backup_dir}" -} - -normalize_owlbot_yaml_test() { - local temp_dir=$(mktemp -d) - local input_file="${temp_dir}/input.yaml" - local output_file="${temp_dir}/output.yaml" - - cat < "${input_file}" -deep-remove-regex: -- "/java-accesscontextmanager/proto-google-.*/src" - - "/java-accesscontextmanager/samples/snippets/generated" - - "/.*google-.*/src/main/java/.*/stub/Version.java" -EOF - - normalize_owlbot_yaml "${input_file}" "${output_file}" "java-accesscontextmanager" - - # Verify content - local expected_content - expected_content=$(cat < GenerationConfig: - return GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[], - ) - - @staticmethod - def __get_an_empty_library_config() -> LibraryConfig: - return LibraryConfig( - api_shortname="", - name_pretty="", - api_description="", - product_documentation="", - gapic_configs=[], - ) diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/model/__init__.py b/sdk-platform-java/hermetic_build/library_generation/tests/model/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/model/repo_config_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/model/repo_config_unit_tests.py deleted file mode 100644 index 12d28fe254df..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/model/repo_config_unit_tests.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest - -from library_generation.model.repo_config import RepoConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) -versions_file = os.path.join(script_dir, "..", "resources", "misc", "versions.txt") - - -class RepoConfigTest(unittest.TestCase): - def test_get_library_versions_with_existing_library(self): - repo_config = RepoConfig( - output_folder="test", libraries=dict(), versions_file=versions_file - ) - self.assertEqual( - "2.25.0", - repo_config.get_library_version("gapic-generator-java"), - ) - self.assertEqual( - "2.16.0", - repo_config.get_library_version("api-common"), - ) - self.assertEqual( - "2.33.0", - repo_config.get_library_version("gax"), - ) - self.assertEqual( - "2.34.0", - repo_config.get_library_version("gax-grpc"), - ) - self.assertEqual( - "0.118.0", - repo_config.get_library_version("gax-httpjson"), - ) - - def test_get_library_versions_with_new_library(self): - repo_config = RepoConfig( - output_folder="test", libraries=dict(), versions_file=versions_file - ) - self.assertEqual( - "0.0.0", - repo_config.get_library_version("example-artifact"), - ) diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/compare_poms.py b/sdk-platform-java/hermetic_build/library_generation/tests/owlbot/compare_poms.py similarity index 97% rename from sdk-platform-java/hermetic_build/library_generation/tests/compare_poms.py rename to sdk-platform-java/hermetic_build/library_generation/tests/owlbot/compare_poms.py index b4d2947166fc..aff8c2ffff77 100644 --- a/sdk-platform-java/hermetic_build/library_generation/tests/compare_poms.py +++ b/sdk-platform-java/hermetic_build/library_generation/tests/owlbot/compare_poms.py @@ -5,7 +5,11 @@ There is a special case for `dependency`, where the maven coordinates are prepared as well """ -from library_generation.utils.utilities import eprint +import sys + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + import xml.etree.ElementTree as et from collections import Counter import sys diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py index a284d837ec4b..61abcd76cbb5 100644 --- a/sdk-platform-java/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py +++ b/sdk-platform-java/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py @@ -15,7 +15,7 @@ import shutil import unittest from library_generation.owlbot.src.fix_poms import main -from library_generation.tests.compare_poms import compare_pom_in_subdir +from library_generation.tests.owlbot.compare_poms import compare_pom_in_subdir script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "..", "resources", "test-owlbot") diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/QueryServiceGrpc_copy.java b/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/QueryServiceGrpc_copy.java deleted file mode 100644 index cf2bf2b0eedc..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/QueryServiceGrpc_copy.java +++ /dev/null @@ -1,9 +0,0 @@ -/** - * This file is only used in testing `remove_grpc_version` in utilities.sh. - */ -@javax.annotation.processing.Generated( - value = "by gRPC proto compiler 1.55.1", - comments = "Source: google/monitoring/v3/query_service.proto") -public final class QueryServiceGrpc_copy { - -} diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example.yaml deleted file mode 100644 index 7bd824910ecd..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.legacy.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.legacy.yaml deleted file mode 100644 index 7da8426f34a5..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.legacy.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.yaml deleted file mode 100644 index 7bd824910ecd..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic_legacy.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic_legacy.yaml deleted file mode 100644 index 7da8426f34a5..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic_legacy.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_grpc_service_config.json b/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_grpc_service_config.json deleted file mode 100644 index 947c39083510..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/gapic_options/example_grpc_service_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "comment": "this file is only used in testing `get_gapic_opts` in utilities.sh" -} diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml deleted file mode 100644 index 225b4620bf5f..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -deep-remove-regex: -- "/java-bare-metal-solution/grpc-google-.*/src" -- "/java-bare-metal-solution/proto-google-.*/src" -- "/java-bare-metal-solution/google-.*/src" -- "/java-bare-metal-solution/samples/snippets/generated" - -deep-preserve-regex: -- "/java-bare-metal-solution/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" - -deep-copy-regex: -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/proto-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/grpc-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/google-cloud-bare-metal-solution/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/samples/snippets/generated" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/samples/snippets/generated" - -api-name: baremetalsolution \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json b/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json deleted file mode 100644 index 2cccd4b8894a..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "api_shortname": "secretmanager", - "name_pretty": "Secret Management", - "product_documentation": "https://cloud.google.com/solutions/secrets-management/", - "api_description": "allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-secretmanager/latest/overview", - "release_level": "preview", - "transport": "http", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-secretmanager", - "distribution_name": "com.google.cloud:google-cloud-secretmanager", - "api_id": "secretmanager.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true -} \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json b/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json deleted file mode 100644 index 0b3bbc2b55eb..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json b/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json deleted file mode 100644 index b39c297a852f..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/java-bare-metal-solution", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_COMBO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "extra_versioned_modules": "test-module", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json b/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json deleted file mode 100644 index 6c54c6ce346f..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "library_type": "OTHER", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py b/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py deleted file mode 100644 index fbb37df70f1a..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import synthtool as s -from synthtool.languages import java - -for library in s.get_staging_dirs(): - # put any special-case replacements here - s.move(library) - -s.remove_staging_dirs() -java.common_templates( - monorepo=True, - excludes=[ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore" -]) \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/generation_config.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/generation_config.yaml deleted file mode 100644 index d71f3863ecde..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/generation_config.yaml +++ /dev/null @@ -1,44 +0,0 @@ -gapic_generator_version: 2.38.1 -googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." - release_level: "stable" - library_name: "apigee-connect" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 - - - api_shortname: alloydb - name_pretty: AlloyDB - product_documentation: https://cloud.google.com/alloydb/ - api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service - with industry-leading performance, availability, and scale. - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/v1 - - proto_path: google/cloud/alloydb/v1alpha - - proto_path: google/cloud/alloydb/v1beta - - - api_shortname: alloydb - name_pretty: AlloyDB connectors - product_documentation: https://cloud.google.com/alloydb/docs - api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for - demanding transactional workloads. It provides enterprise-grade performance and - availability while maintaining 100% compatibility with open-source PostgreSQL. - library_name: alloydb-connectors - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/connectors/v1 - - proto_path: google/cloud/alloydb/connectors/v1alpha - - proto_path: google/cloud/alloydb/connectors/v1beta - - - api_shortname: cloudcontrolspartner - name_pretty: Cloud Controls Partner API - product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners - api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. - GAPICs: - - proto_path: google/cloud/cloudcontrolspartner/v1 - - proto_path: google/cloud/cloudcontrolspartner/v1beta diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/pr-description-golden.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/pr-description-golden.txt deleted file mode 100644 index cb855dcd5221..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/pr-description-golden.txt +++ /dev/null @@ -1,54 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@a17d4ca](https://github.com/googleapis/googleapis/commit/a17d4caf184b050d50cacf2b0d579ce72c31ce74) (exclusive) and [googleapis/googleapis@4ce0ff6](https://github.com/googleapis/googleapis/commit/4ce0ff67a3d4509be641cbe47a35844ddc1268fc) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -docs: [cloudcontrolspartner] update documentation URL - -PiperOrigin-RevId: 612723053 - -Source Link: [googleapis/googleapis@7659dd2](https://github.com/googleapis/googleapis/commit/7659dd2244563fd902b681bdcadbe5385b8d3462) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [cloudcontrolspartner] added CloudControlsPartner API - -PiperOrigin-RevId: 612632640 - -Source Link: [googleapis/googleapis@05d889e](https://github.com/googleapis/googleapis/commit/05d889e7dfe087fc2ddc9de9579f01d4e1c2f35e) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [alloydb] support for obtaining the public IP address of an Instance -feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API -feat: [alloydb] add PSC cluster and instance configuration settings to enable/disable PSC and obtain the PSC endpoint name -feat: [alloydb] add new API to list the databases in a project and location -docs: [alloydb] clarified read pool config is for read pool type instances - -PiperOrigin-RevId: 610475013 - -Source Link: [googleapis/googleapis@aa16fda](https://github.com/googleapis/googleapis/commit/aa16fdad909bc33e2d4ff04cfde56a46d0e52b13) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [alloydb] support for obtaining the public IP address of an Instance -feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API - -PiperOrigin-RevId: 610415824 - -Source Link: [googleapis/googleapis@0733fdb](https://github.com/googleapis/googleapis/commit/0733fdb5f745192f9f3c95f8d08039286567cbcc) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -docs: [cloudcontrolspartner] Updated comment for method `ListCustomers` in service `CloudControlsPartnerCore` -docs: [cloudcontrolspartner] Updated comment for field `location` in message `.google.cloud.cloudcontrolspartner.v1beta.Workload` -docs: [cloudcontrolspartner] Updated a comment for field `partner_project_id` in message `.google.cloud.cloudcontrolspartner.v1beta.Partner` -docs: [cloudcontrolspartner] Updated documentation URL - -PiperOrigin-RevId: 609026905 - -Source Link: [googleapis/googleapis@9e35c62](https://github.com/googleapis/googleapis/commit/9e35c620157d7b11cb5b2e5d0249c5caaee824f3) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [cloudcontrolspartner] added CloudControlsPartner API - -PiperOrigin-RevId: 606720708 - -Source Link: [googleapis/googleapis@36dedd0](https://github.com/googleapis/googleapis/commit/36dedd0d9020c19d1c8259003c2fe9656ada7471) -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/java-bigtable/generation_config.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/java-bigtable/generation_config.yaml deleted file mode 100644 index c6912c8125e3..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/java-bigtable/generation_config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -gapic_generator_version: 2.37.0 -protoc_version: 25.2 -googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc -libraries: -- api_shortname: bigtable - name_pretty: Cloud Bigtable - api_description: "Java idiomatic client for Cloud Bigtable." - product_documentation: "https://cloud.google.com/bigtable" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-bigtable/latest/history" - issue_tracker: "https://issuetracker.google.com/savedsearches/559777" - release_level: "stable" - language: "java" - repo: "googleapis/java-bigtable" - repo_short: "java-bigtable" - distribution_name: "com.google.cloud:google-cloud-bigtable" - excluded_poms: "google-cloud-bigtable-bom" - extra_versioned_modules: "google-cloud-bigtable-emulator,google-cloud-bigtable-emulator-core" - codeowner_team: "@googleapis/api-bigtable @googleapis/api-bigtable-partners" - library_type: GAPIC_COMBO - GAPICs: - - proto_path: google/bigtable/admin/v2 - - proto_path: google/bigtable/v2 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/java-bigtable/pr-description-golden.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/java-bigtable/pr-description-golden.txt deleted file mode 100644 index 2dcdbc77561e..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/integration/java-bigtable/pr-description-golden.txt +++ /dev/null @@ -1,18 +0,0 @@ -This pull request is generated with proto changes between googleapis commit 679060c64136e85b52838f53cfe612ce51e60d1d (exclusive) and fc3043ebe12fb6bc1729c175e1526c859ce751d8 (inclusive). -Qualified commits are: -[googleapis/googleapis@fbcfef0](https://github.com/googleapis/googleapis/commit/fbcfef09510b842774530989889ed1584a8b5acb) -[googleapis/googleapis@63d2a60](https://github.com/googleapis/googleapis/commit/63d2a60056ad5b156c05c7fb13138fc886c3b739) -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix: extend timeouts for deleting snapshots, backups and tables - -PiperOrigin-RevId: 605388988 - -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update retry settings for backup rpcs - -PiperOrigin-RevId: 605367937 - -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/misc/versions.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/misc/versions.txt deleted file mode 100644 index e4258504e423..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/misc/versions.txt +++ /dev/null @@ -1,9 +0,0 @@ -# test versions.txt obtained from sdk-platform-java - -gapic-generator-java:2.25.0:2.25.1-SNAPSHOT -api-common:2.16.0:2.16.1-SNAPSHOT -gax:2.33.0:2.33.1-SNAPSHOT -gax-grpc:2.34.0:2.33.1-SNAPSHOT -gax-httpjson:0.118.0:0.118.1-SNAPSHOT -proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT -grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto b/sdk-platform-java/hermetic_build/library_generation/tests/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-1/fake.proto b/sdk-platform-java/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-1/fake.proto deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-2/fake.proto b/sdk-platform-java/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-2/fake.proto deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config.yaml deleted file mode 100644 index 168c8fd9a533..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - codeowner_team: "@googleapis/analytics-dpe" - excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 - excluded_dependencies: google-iam-policy - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config_library_modified.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config_library_modified.yaml deleted file mode 100644 index f9ae96693bb3..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config_library_modified.yaml +++ /dev/null @@ -1,14 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml deleted file mode 100644 index c5613f4308ff..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml +++ /dev/null @@ -1,51 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 -template_excludes: - - ".github/*" - - ".kokoro/*" - - "samples/*" - - "CODE_OF_CONDUCT.md" - - "CONTRIBUTING.md" - - "LICENSE" - - "SECURITY.md" - - "java.header" - - "license-checks.xml" - - "renovate.json" - - ".gitignore" -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - codeowner_team: "@googleapis/analytics-dpe" - excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 - excluded_dependencies: google-iam-policy - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - - api_shortname: another-cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_baseline.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_baseline.yaml deleted file mode 100644 index c2c4fd4a3be8..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_baseline.yaml +++ /dev/null @@ -1,30 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: preview - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_current.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_current.yaml deleted file mode 100644 index 3ee2c8be2ca7..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_current.yaml +++ /dev/null @@ -1,30 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: stable - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_with_common_protos.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_with_common_protos.yaml deleted file mode 100644 index 6d4c94444a21..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_with_common_protos.yaml +++ /dev/null @@ -1,43 +0,0 @@ -googleapis_commitish: 6a474b31c53cc1797710206824a17b364a835d2d -gapic_generator_version: 2.34.0 -# the libraries are ordered with respect to library name, which is -# java-{library.library_name} or java-{library.api-shortname} when -# library.library_name is not defined. -libraries: -- api_shortname: common-protos - name_pretty: Common Protos - product_documentation: https://github.com/googleapis/api-common-protos - api_description: Protobuf classes for Google's common protos. - release_level: stable - client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history - distribution_name: com.google.api.grpc:proto-google-common-protos - excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" - excluded_poms: "proto-google-common-protos-bom,proto-google-common-protos" - library_type: OTHER - GAPICs: - - proto_path: google/api - - proto_path: google/apps/card/v1 - - proto_path: google/cloud - - proto_path: google/cloud/audit - - proto_path: google/cloud/location - - proto_path: google/geo/type - - proto_path: google/logging/type - - proto_path: google/longrunning - - proto_path: google/rpc - - proto_path: google/rpc/context - - proto_path: google/shopping/type - - proto_path: google/type -- api_shortname: iam - name_pretty: IAM - product_documentation: https://cloud.google.com/iam - api_description: Manages access control for Google Cloud Platform resources - release_level: stable - client_documentation: https://cloud.google.com/java/docs/reference/proto-google-iam-v1/latest/overview - distribution_name: com.google.api.grpc:proto-google-iam-v1 - excluded_dependencies: "grpc-google-iam-v1" - excluded_poms: "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1" - library_type: OTHER - GAPICs: - - proto_path: google/iam/v1 - - proto_path: google/iam/v2 - - proto_path: google/iam/v2beta \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_without_common_protos.yaml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_without_common_protos.yaml deleted file mode 100644 index ca21ccfb0186..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test-config/monorepo_without_common_protos.yaml +++ /dev/null @@ -1,33 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml deleted file mode 100644 index 304ee9b8921e..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - 4.0.0 - com.google.cloud - gapic-libraries-bom - pom - 1.29.0-SNAPSHOT - Google Cloud Java BOM - - BOM for the libraries in google-cloud-java repository. Users should not - depend on this artifact explicitly because this BOM is an implementation - detail of the Libraries BOM. - - - - google-cloud-pom-parent - com.google.cloud - 1.29.0-SNAPSHOT - ../google-cloud-pom-parent/pom.xml - - - - - - com.google.cloud - google-cloud-dns - 2.33.0-SNAPSHOT - - - com.google.cloud - google-cloud-service-control-bom - 1.35.0-SNAPSHOT - pom - import - - - com.google.cloud - google-cloud-tasks-bom - 2.35.0-SNAPSHOT - pom - import - - - - \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-dns/pom.xml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-dns/pom.xml deleted file mode 100644 index 5ce8b224d521..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-dns/pom.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-dns - jar - 2.33.0-SNAPSHOT - Google Cloud DNS Parent - diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml deleted file mode 100644 index 640bd2f989bd..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-service-control-bom - 1.35.0-SNAPSHOT - pom - diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml deleted file mode 100644 index c327f8af7e88..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-tasks-bom - 2.35.0-SNAPSHOT - pom - diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/pom-golden.xml b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/pom-golden.xml deleted file mode 100644 index d7c0a1f6a124..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/pom-golden.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - 4.0.0 - google-cloud-java - com.google.cloud - 0.201.0 - pom - - - true - - - - gapic-libraries-bom - google-cloud-jar-parent - google-cloud-pom-parent - java-dns - java-service-control - java-tasks - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.1.3 - - true - - - - - - - - release-staging-repository - - - - !gpg.executable - - - - - sonatype-nexus-snapshots - https://google.oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-staging - https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.7.0 - true - - sonatype-nexus-staging - https://google.oss.sonatype.org/ - false - - - - - - - release-non-google-oss-sonatype - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - ossrh - https://oss.sonatype.org/ - - - - - - - \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/versions.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/versions.txt deleted file mode 100644 index 6a537f4a3989..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/versions.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Format: -# module:released-version:current-version - -google-cloud-java:1.28.0:1.29.0-SNAPSHOT diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt b/sdk-platform-java/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/test_utilities.sh b/sdk-platform-java/hermetic_build/library_generation/tests/test_utilities.sh deleted file mode 100755 index bbcdee1ebc12..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/test_utilities.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash - -set -xeo pipefail -test_utilities_script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -# Utility functions commonly used in test cases. - -# Variables used to generate final result -total_num=0 -succeed_num=0 -failed_num=0 -failed_tests="" - -############# Helper functions, they shouldn't be called outside this file ############# -__test_executed() { - total_num=$((1 + total_num)) -} - -__test_succeed() { - succeed_num=$((1 + succeed_num)) -} - -__test_failed() { - failed_test=$1 - failed_num=$((1 + failed_num)) - failed_tests="${failed_tests} ${failed_test}" -} - - -############# Functions used in test execution. They can only be called once per test ############# - -assertEquals() { - local expected=$1 - local actual=$2 - if [[ "${expected}" == "${actual}" ]]; then - __test_succeed - return - fi - - echo "Error: expected ${expected}, got ${actual} instead." - __test_failed "${ut}" -} - -assertFileOrDirectoryExists() { - local expected_file=$1 - if [ -d "${expected_file}" ]|| [ -f "${expected_file}" ]; then - __test_succeed - return - fi - - echo "Error: ${expected_file} does not exist." - __test_failed "${ut}" -} - -# Clean up generated files, tooling when testing `generate_library.sh`. -cleanup() { - local library_directory=$1 - rm -rf ../"${library_directory}" \ - google/protobuf \ - protobuf-* \ - gapic-generator-java-*.jar \ - gapic-generator-java-pom-parent-*.pom \ - protoc-gen-grpc-*.exe -} - -execute_tests() { - local test_list=("$@") - for ut in "${test_list[@]}"; do - echo "========== Execute ${ut} ==========" - __test_executed - "${ut}" - done - - echo "Test result: ${total_num} tests executed, ${succeed_num} succeed, ${failed_num} failed." - if [[ "${total_num}" == "${succeed_num}" ]]; then - echo "All tests passed." - exit - fi - - echo "Test failed." - echo "Failed test(s): ${failed_tests}." - exit 1 -} - -############# Utility functions used in `generate_library_integration_tests.sh` ############# - -# Obtains a version from a bazel WORKSPACE file -# -# versions look like "_ggj_version="1.2.3" -# It will return 1.2.3 for such example - -# performs a deep structural comparison between the current pom in a git -# folder and the one at HEAD. -# This function is OS-dependent, so it sources the main utilities script to -# perform detection -compare_poms() { - target_dir=$1 - source "${test_utilities_script_dir}/../utilities.sh" - os_architecture=$(detect_os_architecture) - pushd "${target_dir}" &> /dev/null - find . -name 'pom.xml' -exec cp {} {}.new \; - find . -name 'pom.xml' -exec git checkout HEAD -- {} \; - # compare_poms.py exits with non-zero if diffs are found - set -e - result=0 - if [ "${os_architecture}" == "linux-x86_64" ]; then - find . -name 'pom.xml' -print0 | xargs -i -0 python3 "${test_utilities_script_dir}/compare_poms.py" {} {}.new false || result=$? - else - find . -name 'pom.xml' -print0 | xargs -I{} -0 python3 "${test_utilities_script_dir}/compare_poms.py" {} {}.new false || result=$? - fi - popd &> /dev/null # target_dir - echo ${result} -} - -# computes the `destination_path` variable by inspecting the contents of the -# googleapis-gen at $proto_path. -compute_destination_path() { - local proto_path=$1 - local output_folder=$2 - pushd "${output_folder}" &> /dev/null - local destination_path=$(find "googleapis-gen/${proto_path}" -maxdepth 1 -name 'google-*-java' \ - | rev \ - | cut -d'/' -f1 \ - | rev - ) - popd &> /dev/null # output_folder - echo "${destination_path}" -} - diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/test_utils.py b/sdk-platform-java/hermetic_build/library_generation/tests/test_utils.py deleted file mode 100644 index 890cd95362b8..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/test_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest -from difflib import unified_diff -from pathlib import Path - -from typing import List - - -class FileComparator(unittest.TestCase): - def compare_files(self, expect: str, actual: str): - with open(expect, "r") as f: - expected_lines = f.readlines() - with open(actual, "r") as f: - actual_lines = f.readlines() - - diff = list(unified_diff(expected_lines, actual_lines)) - self.assertEqual( - first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff) - ) - - -def cleanup(files: List[str]): - for file in files: - path = Path(file).resolve() - if path.is_file(): - path.unlink() - elif path.is_dir(): - path.rmdir() diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/utilities_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/utilities_unit_tests.py deleted file mode 100644 index 0708fdb4c556..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/utilities_unit_tests.py +++ /dev/null @@ -1,401 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Unit tests for python scripts -""" - -import shutil -import unittest -import os -import io -import contextlib -from pathlib import Path -from library_generation.utils import utilities as util -from common.model.gapic_config import GapicConfig -from common.model.gapic_inputs import GapicInputs -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig -from library_generation.tests.test_utils import FileComparator -from library_generation.tests.test_utils import cleanup - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "resources") -file_comparator = FileComparator() -library_1 = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - gapic_configs=list(), - library_name="bare-metal-solution", - rest_documentation="https://cloud.google.com/bare-metal/docs/reference/rest", - rpc_documentation="https://cloud.google.com/bare-metal/docs/reference/rpc", - recommended_package="com.google.example", - min_java_version=8, -) -library_2 = LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret Management", - product_documentation="https://cloud.google.com/solutions/secrets-management/", - api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - gapic_configs=list(), -) -common_protos = LibraryConfig( - api_shortname="common-protos", - name_pretty="Common Protos", - product_documentation="", - api_description="example description", - gapic_configs=list(), -) -test_library_with_custom_transport = LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret Management", - product_documentation="https://cloud.google.com/solutions/secrets-management/", - api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - gapic_configs=list(), - transport="rest", -) - - -class UtilitiesTest(unittest.TestCase): - """ - Unit tests for utilities.py - """ - - CONFIGURATION_YAML_PATH = os.path.join( - script_dir, - "resources", - "integration", - "google-cloud-java", - "generation_config.yaml", - ) - - def test_create_argument_valid_container_succeeds(self): - container_value = "google/test/v1" - container = GapicConfig(container_value) - argument_key = "proto_path" - result = util.create_argument(argument_key, container) - self.assertEqual([f"--{argument_key}", container_value], result) - - def test_create_argument_empty_container_returns_empty_list(self): - container = dict() - argument_key = "proto_path" - result = util.create_argument(argument_key, container) - self.assertEqual([], result) - - def test_create_argument_none_container_fails(self): - container = None - argument_key = "proto_path" - result = util.create_argument(argument_key, container) - self.assertEqual([], result) - - def test_sh_util_existent_function_succeeds(self): - result = util.sh_util("extract_folder_name path/to/folder_name") - self.assertEqual("folder_name", result) - - def test_sh_util_nonexistent_function_fails(self): - with self.assertRaises(RuntimeError): - result = util.sh_util("nonexistent_function") - - def test_mv_src_files_gapic_main_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/gapic") - os.environ["folder_name"] = "example" - util.sh_util("mv_src_files gapic main destination") - self.assertTrue( - os.path.isfile("destination/gapic-example/src/main/java/example_main.txt") - ) - shutil.rmtree("destination/gapic-example") - os.chdir(previous_dir) - - def test_mv_src_files_gapic_test_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/gapic") - os.environ["folder_name"] = "example" - util.sh_util("mv_src_files gapic test destination") - self.assertTrue( - os.path.isfile("destination/gapic-example/src/test/java/example_test.txt") - ) - shutil.rmtree("destination/gapic-example") - os.chdir(previous_dir) - - def test_mv_src_files_proto_main_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/proto") - os.environ["folder_name"] = "example" - util.sh_util("mv_src_files proto main destination") - self.assertTrue( - os.path.isfile( - "destination/proto-example/src/main/java/example_proto_main.txt" - ) - ) - shutil.rmtree("destination/proto-example") - os.chdir(previous_dir) - - def test_mv_src_files_sample_suffix_io_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/samples") - util.sh_util("mv_src_files samples main destination_io") - self.assertTrue( - os.path.isfile( - "destination_io/samples/snippets/generated/io/example_io_sample.txt" - ) - ) - shutil.rmtree("destination_io/samples") - os.chdir(previous_dir) - - def test_mv_src_files_sample_suffix_com_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/samples") - util.sh_util("mv_src_files samples main destination_com") - self.assertTrue( - os.path.isfile( - "destination_com/samples/snippets/generated/com/example_com_sample.txt" - ) - ) - shutil.rmtree("destination_com/samples") - os.chdir(previous_dir) - - def test_eprint_valid_input_succeeds(self): - test_input = "This is some test input" - # create a stdio capture object - stderr_capture = io.StringIO() - # run eprint() with the capture object - with contextlib.redirect_stderr(stderr_capture): - util.eprint(test_input) - result = stderr_capture.getvalue() - # print() appends a `\n` each time it's called - self.assertEqual(test_input + "\n", result) - - def test_get_library_repository_with_common_protos_returns_sdk_platform_java(self): - config = self.__get_a_gen_config(3) - library = common_protos - result = util.get_library_repository(config, library) - self.assertEqual("googleapis/google-cloud-java", result) - - def test_get_library_repository_with_monorepo_returns_google_cloud_java(self): - config = self.__get_a_gen_config(2) - library = library_1 - result = util.get_library_repository(config, library) - self.assertEqual("googleapis/google-cloud-java", result) - - def test_get_library_repository_with_split_repo_returns_library_repo(self): - config = self.__get_a_gen_config(1) - library = library_1 - result = util.get_library_repository(config, library) - self.assertEqual("googleapis/java-bare-metal-solution", result) - - def test_generate_postprocessing_prerequisite_files_non_monorepo_success(self): - library_path = self.__setup_postprocessing_prerequisite_files( - combination=1, library_type="GAPIC_COMBO" - ) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-non-monorepo-golden.json", - ) - # since this is a single library, we treat this as HW repository, - # meaning that the owlbot yaml will be inside a .github folder - file_comparator.compare_files( - f"{library_path}/.github/.OwlBot-hermetic.yaml", - f"{library_path}/.OwlBot-hermetic-golden.yaml", - ) - file_comparator.compare_files( - f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" - ) - self.__remove_postprocessing_prerequisite_files( - path=library_path, is_monorepo=False - ) - - def test_generate_postprocessing_prerequisite_files_monorepo_success(self): - library_path = self.__setup_postprocessing_prerequisite_files(combination=2) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-monorepo-golden.json", - ) - file_comparator.compare_files( - f"{library_path}/.OwlBot-hermetic.yaml", - f"{library_path}/.OwlBot-hermetic-golden.yaml", - ) - file_comparator.compare_files( - f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" - ) - self.__remove_postprocessing_prerequisite_files(path=library_path) - - def test_generate_postprocessing_prerequisite_files_proto_only_repo_success(self): - library_path = self.__setup_postprocessing_prerequisite_files( - combination=3, library_type="OTHER" - ) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-proto-only-golden.json", - ) - file_comparator.compare_files( - f"{library_path}/.OwlBot-hermetic.yaml", - f"{library_path}/.OwlBot-hermetic-golden.yaml", - ) - file_comparator.compare_files( - f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" - ) - self.__remove_postprocessing_prerequisite_files(path=library_path) - - def test_generate_postprocessing_prerequisite_files__custom_transport_set_in_config__success( - self, - ): - """ - This test generates files for `test_library_with_custom_transport`, which - has an explicit value for transport declared (http). This is expected to - override the value obtained in BUILD.bazel via gapic_inputs.parse(). For - testing purposes, we test with a default GapicInputs object, whose transport - is set to "grpc". - """ - library_path = self.__setup_postprocessing_prerequisite_files( - combination=2, library=test_library_with_custom_transport - ) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-custom-transport-golden.json", - ) - self.__remove_postprocessing_prerequisite_files(path=library_path) - - def test_create__library_invalid_transport__fails( - self, - ): - with self.assertRaises(ValueError): - test_library_with_invalid_transport = LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret Management", - product_documentation="https://cloud.google.com/solutions/secrets-management/", - api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - gapic_configs=list(), - transport="http", - ) - - def test_prepare_repo_monorepo_success(self): - gen_config = self.__get_a_gen_config(2) - repo_config = util.prepare_repo( - gen_config=gen_config, - library_config=gen_config.libraries, - repo_path=f"{resources_dir}/misc", - ) - self.assertEqual("output", Path(repo_config.output_folder).name) - library_path = sorted([Path(key).name for key in repo_config.libraries]) - self.assertEqual( - ["java-bare-metal-solution", "java-secretmanager"], library_path - ) - - def test_prepare_repo_monorepo_failed(self): - gen_config = self.__get_a_gen_config(2) - self.assertRaises( - FileNotFoundError, - util.prepare_repo, - gen_config, - gen_config.libraries, - f"{resources_dir}/non-exist", - ) - - def test_prepare_repo_split_repo_success(self): - gen_config = self.__get_a_gen_config(1) - repo_config = util.prepare_repo( - gen_config=gen_config, - library_config=gen_config.libraries, - repo_path=f"{resources_dir}/misc", - ) - self.assertEqual("output", Path(repo_config.output_folder).name) - library_path = sorted([Path(key).name for key in repo_config.libraries]) - self.assertEqual(["misc"], library_path) - shutil.rmtree(repo_config.output_folder) - - def __setup_postprocessing_prerequisite_files( - self, - combination: int, - library_type: str = "GAPIC_AUTO", - library: LibraryConfig = library_1, - ) -> str: - library_path = f"{resources_dir}/goldens" - files = [ - f"{library_path}/.repo-metadata.json", - f"{library_path}/.OwlBot-hermetic.yaml", - f"{library_path}/owlbot.py", - ] - cleanup(files) - library.library_type = library_type - config = self.__get_a_gen_config(combination, library_type=library_type) - proto_path = "google/cloud/baremetalsolution/v2" - gapic_inputs = GapicInputs() # defaults to transport=grpc - transport = library.get_transport(gapic_inputs) - util.generate_postprocessing_prerequisite_files( - config=config, - library=library, - proto_path=proto_path, - transport=transport, - library_path=library_path, - ) - return library_path - - @staticmethod - def __get_a_gen_config( - combination: int, library_type: str = "GAPIC_AUTO" - ) -> GenerationConfig: - """ - Returns an object of GenerationConfig with one to three of - LibraryConfig objects. Other attributes are set to empty str. - - :param combination: combination of LibraryConfig objects associated with - the GenerationConfig. Only support 1, 2 or 3. - :return: an object of GenerationConfig - """ - if combination == 1: - libraries = [library_1] - elif combination == 2: - libraries = [library_1, library_2] - else: - libraries = [library_1, common_protos] - - # update libraries with custom configuration (for now, only - # library_type) - for library in libraries: - library.library_type = library_type - if combination == 1: - # treat this as a HW library case to generate a real-life - # repo-metadata - library.extra_versioned_modules = "test-module" - else: - library.extra_versioned_modules = None - - return GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=libraries, - ) - - @staticmethod - def __remove_postprocessing_prerequisite_files( - path: str, is_monorepo: bool = True - ) -> None: - os.remove(f"{path}/.repo-metadata.json") - os.remove(f"{path}/owlbot.py") - if is_monorepo: - os.remove(f"{path}/.OwlBot-hermetic.yaml") - return - if os.path.isdir(f"{path}/.github"): - shutil.rmtree(f"{path}/.github", ignore_errors=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/utils/__init__.py b/sdk-platform-java/hermetic_build/library_generation/tests/utils/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/utils/monorepo_postprocessor_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/utils/monorepo_postprocessor_unit_tests.py deleted file mode 100644 index b95708c2ffae..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/utils/monorepo_postprocessor_unit_tests.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest - -from library_generation.tests.test_utils import FileComparator -from library_generation.tests.test_utils import cleanup -from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -file_comparator = FileComparator() - - -class MonorepoPostprocessorTest(unittest.TestCase): - def test_monorepo_postprocessing_valid_repository_success(self): - repository_path = f"{resources_dir}/test_monorepo_postprocessing" - versions_file = f"{repository_path}/versions.txt" - files = [ - f"{repository_path}/pom.xml", - f"{repository_path}/gapic-libraries-bom/pom.xml", - ] - cleanup(files) - monorepo_postprocessing( - repository_path=repository_path, versions_file=versions_file - ) - file_comparator.compare_files( - expect=f"{repository_path}/pom-golden.xml", - actual=f"{repository_path}/pom.xml", - ) - file_comparator.compare_files( - expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml", - actual=f"{repository_path}/gapic-libraries-bom/pom.xml", - ) - cleanup(files) diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/utils/pom_generator_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/utils/pom_generator_unit_tests.py deleted file mode 100644 index 7a829e58aa97..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/utils/pom_generator_unit_tests.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest - -from library_generation.utils.pom_generator import get_version_from - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") - - -class PomGeneratorTest(unittest.TestCase): - def test_get_version_from_returns_current(self): - versions_file = f"{resources_dir}/misc/versions.txt" - artifact = "gax-grpc" - self.assertEqual("2.33.1-SNAPSHOT", get_version_from(versions_file, artifact)) - - def test_get_version_from_returns_released(self): - versions_file = f"{resources_dir}/misc/versions.txt" - artifact = "gax-grpc" - self.assertEqual("2.34.0", get_version_from(versions_file, artifact, True)) diff --git a/sdk-platform-java/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py b/sdk-platform-java/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py deleted file mode 100644 index 76b663c6cf1c..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from pathlib import Path -from library_generation.utils.proto_path_utils import remove_version_from - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() - - -class ProtoPathsUtilsTest(unittest.TestCase): - def test_remove_version_from_returns_non_versioned_path(self): - proto_path = "google/cloud/aiplatform/v1" - self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) - - def test_remove_version_from_returns_self(self): - proto_path = "google/cloud/aiplatform" - self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) diff --git a/sdk-platform-java/hermetic_build/library_generation/utils/file_render.py b/sdk-platform-java/hermetic_build/library_generation/utils/file_render.py deleted file mode 100644 index 5c6844575371..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/utils/file_render.py +++ /dev/null @@ -1,24 +0,0 @@ -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from jinja2 import Environment, FileSystemLoader - -script_dir = os.path.dirname(os.path.realpath(__file__)) -jinja_env = Environment(loader=FileSystemLoader(f"{script_dir}/../templates")) - - -def render(template_name: str, output_name: str, **kwargs): - template = jinja_env.get_template(template_name) - t = template.stream(kwargs) - directory = os.path.dirname(output_name) - if not os.path.isdir(directory): - os.makedirs(directory) - t.dump(str(output_name)) diff --git a/sdk-platform-java/hermetic_build/library_generation/utils/monorepo_postprocessor.py b/sdk-platform-java/hermetic_build/library_generation/utils/monorepo_postprocessor.py deleted file mode 100644 index cd522b83b529..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/utils/monorepo_postprocessor.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from library_generation.utils.pom_generator import generate_gapic_bom -from library_generation.utils.pom_generator import generate_root_pom - - -def monorepo_postprocessing( - repository_path: str, - versions_file: str, -) -> None: - """ - Perform repository level post-processing - :param repository_path: the path of the repository - :param versions_file: the versions_txt contains version of modules - :return: None - """ - generate_root_pom(repository_path=repository_path) - generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) diff --git a/sdk-platform-java/hermetic_build/library_generation/utils/pom_generator.py b/sdk-platform-java/hermetic_build/library_generation/utils/pom_generator.py deleted file mode 100644 index 5b7aaf11077d..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/utils/pom_generator.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from pathlib import Path - -from lxml import etree -from typing import List - -from library_generation.model.bom_config import BomConfig -from library_generation.utils.file_render import render - -project_tag = "{http://maven.apache.org/POM/4.0.0}" -group_id_tag = "groupId" -artifact_tag = "artifactId" -version_tag = "version" - - -def generate_root_pom(repository_path: str) -> None: - print("Regenerating root pom.xml") - modules = __search_for_java_modules(repository_path) - render( - template_name="root-pom.xml.j2", - output_name=f"{repository_path}/pom.xml", - modules=modules, - ) - - -def generate_gapic_bom(repository_path: str, versions_file: str) -> None: - print("Regenerating gapic-libraries-bom") - bom_configs = __search_for_bom_artifact(repository_path) - monorepo_version = get_version_from( - versions_file=versions_file, - artifact_id="google-cloud-java", - ) - render( - template_name="gapic-libraries-bom.xml.j2", - output_name=f"{repository_path}/gapic-libraries-bom/pom.xml", - monorepo_version=monorepo_version, - bom_configs=bom_configs, - ) - - -def get_version_from( - versions_file: str, artifact_id: str, is_released: bool = False -) -> str: - """ - Get version of a given artifact from versions.txt - :param versions_file: the path of version.txt - :param artifact_id: the artifact id - :param is_released: whether returns the released or current version - :return: the version of the artifact - """ - index = 1 if is_released else 2 - with open(versions_file, "r") as f: - for line in f.readlines(): - if artifact_id in line: - return line.split(":")[index].strip() - - -MODULE_ALLOWLIST = set( - [ - "google-auth-library-java", - "sdk-platform-java", - "java-showcase", - "grpc-gcp-java", - ] -) - - -def __search_for_java_modules( - repository_path: str, -) -> List[str]: - repo = Path(repository_path).resolve() - modules = [] - for sub_dir in repo.iterdir(): - if sub_dir.is_dir() and ( - sub_dir.name.startswith("java-") or sub_dir.name in MODULE_ALLOWLIST - ): - modules.append(sub_dir.name) - return sorted(modules) - - -def __search_for_bom_artifact( - repository_path: str, -) -> List[BomConfig]: - repo = Path(repository_path).resolve() - module_exclusions = ["gapic-libraries-bom"] - sub_module_exclusions = ["google-cloud-bigtable-deps-bom"] - group_id_inclusions = [ - "com.google.cloud", - "com.google.analytics", - "com.google.area120", - ] - bom_configs = [] - for module in repo.iterdir(): - if module.is_file() or module.name in module_exclusions: - continue - for sub_module in module.iterdir(): - if ( - sub_module.is_dir() - and sub_module.name.endswith("-bom") - and sub_module.name not in sub_module_exclusions - ): - root = etree.parse(f"{sub_module}/pom.xml").getroot() - group_id = root.find(f"{project_tag}{group_id_tag}").text - if group_id not in group_id_inclusions: - continue - artifact_id = root.find(f"{project_tag}{artifact_tag}").text - version = root.find(f"{project_tag}{version_tag}").text - index = artifact_id.rfind("-") - version_annotation = artifact_id[:index] - bom_configs.append( - BomConfig( - group_id=group_id, - artifact_id=artifact_id, - version=version, - version_annotation=version_annotation, - ) - ) - # handle edge case: java-grafeas - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-grafeas", - group_id="io.grafeas", - artifact_id="grafeas", - ) - # handle edge case: java-dns - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-dns", - group_id="com.google.cloud", - artifact_id="google-cloud-dns", - ) - # handle edge case: java-notification - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-notification", - group_id="com.google.cloud", - artifact_id="google-cloud-notification", - ) - - return sorted(bom_configs) - - -def __handle_special_bom( - repository_path: str, - module: str, - group_id: str, - artifact_id: str, -) -> List[BomConfig]: - pom = f"{repository_path}/{module}/pom.xml" - if not Path(pom).exists(): - return [] - root = etree.parse(pom).getroot() - version = root.find(f"{project_tag}{version_tag}").text - return [ - BomConfig( - group_id=group_id, - artifact_id=artifact_id, - version=version, - version_annotation=artifact_id, - is_import=False, - ) - ] diff --git a/sdk-platform-java/hermetic_build/library_generation/utils/proto_path_utils.py b/sdk-platform-java/hermetic_build/library_generation/utils/proto_path_utils.py deleted file mode 100644 index 27e3f8aa38d0..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/utils/proto_path_utils.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re - - -def remove_version_from(proto_path: str) -> str: - """ - Remove the version of a proto_path - :param proto_path: versioned proto_path - :return: the proto_path without version - """ - version_pattern = "^v[1-9]" - index = proto_path.rfind("/") - version = proto_path[index + 1 :] - if re.match(version_pattern, version): - return proto_path[:index] - return proto_path diff --git a/sdk-platform-java/hermetic_build/library_generation/utils/utilities.py b/sdk-platform-java/hermetic_build/library_generation/utils/utilities.py deleted file mode 100755 index 23fd92458919..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/utils/utilities.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import sys -import subprocess -import os -from pathlib import Path -from typing import Any - -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig -from typing import List -from library_generation.model.repo_config import RepoConfig -from library_generation.utils.file_render import render -from library_generation.utils.proto_path_utils import remove_version_from - -script_dir = os.path.dirname(os.path.realpath(__file__)) - -LIBRARIES_WITHOUT_API_ID = { - "google-auth-library", - "showcase", - "iam", - "api-common", - "common-protos", - "gax", - "core", -} - - -def create_argument(arg_key: str, arg_container: object) -> List[str]: - """ - Generates a list of two elements [argument, value], or returns - an empty array if arg_val is None - """ - arg_val = getattr(arg_container, arg_key, None) - if arg_val is not None: - return [f"--{arg_key}", f"{arg_val}"] - return [] - - -def run_process_and_print_output( - arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs -) -> Any: - """ - Runs a process with the given "arguments" list and prints its output. - If the process fails, then the whole program exits - :param arguments: list of strings or string containing the arguments - :param job_name: optional job name to be printed - :param exit_on_fail: True if the main program should exit when the - subprocess exits with non-zero. Used for testing. - :param kwargs: passed to the underlying subprocess.run() call - """ - # split "arguments" if passed as a single string - if isinstance(arguments, str): - arguments = arguments.split(" ") - if "stderr" not in kwargs: - kwargs["stderr"] = subprocess.STDOUT - proc_info = subprocess.run(arguments, stdout=subprocess.PIPE, **kwargs) - print(proc_info.stdout.decode(), end="", flush=True) - # If stderr was not specified and the process failed, print the error output - if proc_info.returncode != 0 and proc_info.stderr: - print(proc_info.stderr.decode(), end="", flush=True) - print( - f"{job_name} {'finished successfully' if proc_info.returncode == 0 else 'failed'}" - ) - if exit_on_fail and proc_info.returncode != 0: - sys.exit(1) - return proc_info - - -def run_process_and_get_output_string( - arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs -) -> Any: - """ - Wrapper of run_process_and_print_output() that returns the merged - stdout and stderr in a single string without its trailing newline char. - """ - return run_process_and_print_output( - arguments, job_name, exit_on_fail, stderr=subprocess.PIPE, **kwargs - ).stdout.decode()[:-1] - - -def sh_util(statement: str, **kwargs) -> str: - """ - Calls a function defined in library_generation/utils/utilities.sh - """ - if "stdout" not in kwargs: - kwargs["stdout"] = subprocess.PIPE - if "stderr" not in kwargs: - kwargs["stderr"] = subprocess.PIPE - output = "" - with subprocess.Popen( - [ - "bash", - "-exc", - f"source {script_dir}/utilities.sh && {statement}", - ], - **kwargs, - ) as proc: - print("command stderr:") - for line in proc.stderr: - print(line.decode(), end="", flush=True) - print("command stdout:") - for line in proc.stdout: - print(line.decode(), end="", flush=True) - output += line.decode() - proc.wait() - if proc.returncode != 0: - raise RuntimeError( - f"function {statement} failed with exit code {proc.returncode}" - ) - # captured stdout may contain a newline at the end, we remove it - if len(output) > 0 and output[-1] == "\n": - output = output[:-1] - return output - - -def eprint(*args, **kwargs): - """ - prints to stderr - """ - print(*args, file=sys.stderr, **kwargs) - - -def prepare_repo( - gen_config: GenerationConfig, - library_config: List[LibraryConfig], - repo_path: str, - language: str = "java", -) -> RepoConfig: - """ - Gather information of the generated repository. - - :param gen_config: a GenerationConfig object representing a parsed - configuration yaml - :param library_config: a LibraryConfig object contained inside config, - passed here for convenience and to prevent all libraries to be processed - :param repo_path: the path to which the generated repository goes - :param language: programming language of the library - :return: a RepoConfig object contained repository information - :raise FileNotFoundError if there's no versions.txt in repo_path - :raise ValueError if two libraries have the same library_name - """ - output_folder = sh_util("get_output_folder") - print(f"output_folder: {output_folder}") - os.makedirs(output_folder, exist_ok=True) - libraries = {} - for library in library_config: - library_name = f"{language}-{library.get_library_name()}" - library_path = ( - f"{repo_path}/{library_name}" - if gen_config.is_monorepo() - else f"{repo_path}" - ) - # use absolute path because docker requires absolute path - # in volume name. - absolute_library_path = str(Path(library_path).resolve()) - libraries[absolute_library_path] = library - # remove existing .repo-metadata.json - json_name = ".repo-metadata.json" - if os.path.exists(f"{absolute_library_path}/{json_name}"): - os.remove(f"{absolute_library_path}/{json_name}") - versions_file = Path(repo_path) / "versions.txt" - if not versions_file.exists(): - versions_file.touch() - print(f"Created empty versions file: {versions_file}") - return RepoConfig( - output_folder=output_folder, - libraries=libraries, - versions_file=str(versions_file), - ) - - -def get_library_repository( - config: GenerationConfig, library: LibraryConfig, language: str = "java" -): - """ - Obtains the repository identifier (e.g. googleapis/java-bigtable) depending on - whether it's a monorepo (google-cloud-java or sdk-platform-java if has common-protos) or not. - - :return: string representing the repository - """ - if config.is_monorepo() or config.contains_common_protos(): - repo = "googleapis/google-cloud-java" - else: - repo = f"googleapis/{language}-{library.get_library_name()}" - return repo - - -def generate_postprocessing_prerequisite_files( - config: GenerationConfig, - library: LibraryConfig, - proto_path: str, - transport: str, - library_path: str, - language: str = "java", - has_version: bool = True, -) -> None: - """ - Generates the postprocessing prerequisite files for a library. - - :param config: a GenerationConfig object representing a parsed configuration - yaml - :param library: the library configuration - :param proto_path: the path from the root of googleapis to the location of the service - protos. If the path contains a version, it will be removed - :param transport: transport supported by the library - :param library_path: the path to which the generated file goes - :param language: programming language of the library - :return: None - """ - repo = get_library_repository(config, library) - library_name = library.get_library_name() - artifact_id = library.get_artifact_id() - api_id = ( - library.api_id if library.api_id else f"{library.api_shortname}.googleapis.com" - ) - client_documentation = ( - library.client_documentation - if library.client_documentation - else f"https://cloud.google.com/{language}/docs/reference/{artifact_id}/latest/overview" - ) - - # The mapping is needed because transport in .repo-metadata.json - # is one of grpc, http and both, - if transport == "grpc": - converted_transport = "grpc" - elif transport == "rest": - converted_transport = "http" - else: - converted_transport = "both" - - repo_metadata = { - "api_shortname": library.api_shortname, - "name_pretty": library.name_pretty, - "product_documentation": library.product_documentation, - "api_description": library.api_description, - "client_documentation": client_documentation, - "release_level": library.release_level, - "transport": converted_transport, - "language": language, - "repo": repo, - "repo_short": f"{language}-{library_name}", - "distribution_name": library.get_maven_coordinate(), - } - - if ( - library.get_library_name() not in LIBRARIES_WITHOUT_API_ID - and library.library_type != "OTHER" - ): - repo_metadata["api_id"] = api_id - - repo_metadata["library_type"] = library.library_type - repo_metadata["requires_billing"] = library.requires_billing - - if library.api_reference: - repo_metadata["api_reference"] = library.api_reference - if library.codeowner_team: - repo_metadata["codeowner_team"] = library.codeowner_team - if library.excluded_dependencies: - repo_metadata["excluded_dependencies"] = library.excluded_dependencies - if library.excluded_poms: - repo_metadata["excluded_poms"] = library.excluded_poms - if library.issue_tracker: - repo_metadata["issue_tracker"] = library.issue_tracker - if library.rest_documentation: - repo_metadata["rest_documentation"] = library.rest_documentation - if library.rpc_documentation: - repo_metadata["rpc_documentation"] = library.rpc_documentation - if library.extra_versioned_modules: - repo_metadata["extra_versioned_modules"] = library.extra_versioned_modules - if library.recommended_package: - repo_metadata["recommended_package"] = library.recommended_package - if library.min_java_version: - repo_metadata["min_java_version"] = library.min_java_version - - # generate .repo-meta.json - json_file = ".repo-metadata.json" - # .repo-metadata.json is removed before generating the first version of - # a library. This check ensures no duplicated generation. - if not os.path.exists(f"{library_path}/{json_file}"): - with open(f"{library_path}/{json_file}", "w") as fp: - json.dump(repo_metadata, fp, indent=2) - - # generate .OwlBot-hermetic.yaml - owlbot_yaml_file = ".OwlBot-hermetic.yaml" - path_to_owlbot_yaml_file = ( - f"{library_path}/{owlbot_yaml_file}" - if config.is_monorepo() - else f"{library_path}/.github/{owlbot_yaml_file}" - ) - if not os.path.exists(path_to_owlbot_yaml_file): - render( - template_name="owlbot.yaml.monorepo.j2", - output_name=path_to_owlbot_yaml_file, - artifact_id=artifact_id, - proto_path=remove_version_from(proto_path), - module_name=repo_metadata["repo_short"], - api_shortname=library.api_shortname, - has_version=has_version, - ) - - # generate owlbot.py - py_file = "owlbot.py" - template_excludes = [ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore", - ] - if not os.path.exists(f"{library_path}/{py_file}"): - render( - template_name="owlbot.py.j2", - output_name=f"{library_path}/{py_file}", - should_include_templates=True, - template_excludes=template_excludes, - ) diff --git a/sdk-platform-java/hermetic_build/library_generation/utils/utilities.sh b/sdk-platform-java/hermetic_build/library_generation/utils/utilities.sh deleted file mode 100755 index ac0bc946dd65..000000000000 --- a/sdk-platform-java/hermetic_build/library_generation/utils/utilities.sh +++ /dev/null @@ -1,396 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail -utilities_script_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") - -# Utility functions used in `generate_library.sh` and showcase generation. -extract_folder_name() { - local destination_path=$1 - local folder_name=${destination_path##*/} - echo "${folder_name}" -} - -remove_empty_files() { - local category=$1 - local destination_path=$2 - local file_num - find "${destination_path}/${category}-${folder_name}/src/main/java" -type f -size 0 | while read -r f; do rm -f "${f}"; done - # remove the directory if the directory has no files. - file_num=$(find "${destination_path}/${category}-${folder_name}" -type f | wc -l | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - if [[ "${file_num}" == 0 ]]; then - rm -rf "${destination_path}/${category}-${folder_name}" - fi - - if [ -d "${destination_path}/${category}-${folder_name}/src/main/java/samples" ]; then - mv "${destination_path}/${category}-${folder_name}/src/main/java/samples" "${destination_path}/${category}-${folder_name}" - fi -} - -# Move generated files to folders in destination_path. -mv_src_files() { - local category=$1 # one of gapic, proto, samples - local type=$2 # one of main, test - local destination_path=$3 - if [ "${category}" == "samples" ]; then - src_suffix="samples/snippets/generated/src/main/java" - folder_suffix="samples/snippets/generated" - mkdir -p "${destination_path}/${folder_suffix}" - cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}"/* "${destination_path}/${folder_suffix}" - elif [ "${category}" == "proto" ]; then - src_suffix="${category}/src/${type}/java" - folder_suffix="${category}-${folder_name}/src/${type}" - else - src_suffix="src/${type}" - folder_suffix="${category}-${folder_name}/src" - fi - - if [ "${category}" == "samples" ]; then - return - fi - - mkdir -p "${destination_path}/${folder_suffix}" - cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}" "${destination_path}/${folder_suffix}" - rm -r -f "${destination_path}/${folder_suffix}/java/META-INF" -} - -# unzip jar file -unzip_src_files() { - local category=$1 - local destination_path=$2 - local jar_file=java_${category}.jar - mkdir -p "${destination_path}/${category}-${folder_name}/src/main/java" - unzip -q -o "${destination_path}/${jar_file}" -d "${destination_path}/${category}-${folder_name}/src/main/java" - rm -r -f "${destination_path}/${category}-${folder_name}/src/main/java/META-INF" -} - -# get gapic options from .yaml and .json files from proto_path. -get_gapic_opts() { - local transport=$1 - local rest_numeric_enums=$2 - local gapic_yaml=$3 - local service_config=$4 - local service_yaml=$5 - local repo=$6 - local artifact=$7 - local generate_version_java=$8 - - local opts="transport=${transport}" - - if [ "${rest_numeric_enums}" == "true" ]; then - opts="${opts},rest-numeric-enums" - fi - - if [ "${generate_version_java}" == "true" ]; then - opts="${opts},generate-version-java" - fi - - # If any of the gapic options is empty (default value), try to search for - # it in proto_path. - if [[ "${gapic_yaml}" == "" ]]; then - gapic_yaml=$(find "${proto_path}" -type f -name "*gapic.yaml") - fi - - if [[ "${service_config}" == "" ]]; then - service_config=$(find "${proto_path}" -type f -name "*service_config.json") - fi - - if [[ "${service_yaml}" == "" ]]; then - service_yaml=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic*.yaml" \)) - fi - - opts="${opts},grpc-service-config=${service_config},gapic-config=${gapic_yaml},api-service-config=${service_yaml},repo=${repo},artifact=${artifact}" - echo "${opts}" -} - -remove_grpc_version() { - local destination_path=$1 - find "${destination_path}" -type f -name "*Grpc.java" -exec \ - sed -i.bak 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \; -exec rm {}.bak \; -} - -download_protoc() { - local protoc_version=$1 - local os_architecture=$2 - - download_from \ - "https://github.com/protocolbuffers/protobuf/releases/download/v${protoc_version}/protoc-${protoc_version}-${os_architecture}.zip" \ - "protoc-${protoc_version}.zip" \ - "GitHub" - unzip -o -q "protoc-${protoc_version}.zip" - rm -f "protoc-${protoc_version}.zip" "readme.txt" -} - -download_grpc_plugin() { - local grpc_version=$1 - local os_architecture=$2 - grpc_filename="protoc-gen-grpc-java.exe" - # download protoc-gen-grpc-java plugin from Google maven central mirror. - download_from \ - "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/${grpc_version}/protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" \ - "${grpc_filename}" - chmod +x "${grpc_filename}" -} - -download_from() { - local url=$1 - local save_as=$2 - local repo=$3 - # fail-fast, 30 seconds at most, retry 2 times - curl -LJ -o "${save_as}" --fail -m 30 --retry 2 "$url" || download_fail "${save_as}" "${repo}" -} - -# copies the specified file in $1 to $2 -# will return "true" if the copy was successful -copy_from() { - local local_repo=$1 - local save_as=$2 - copy_successful=$(cp "${local_repo}" "${save_as}" && echo "true" || echo "false") - echo "${copy_successful}" -} - -download_fail() { - local artifact=$1 - local repo=${2:-"maven central mirror"} - >&2 echo "Fail to download ${artifact} from ${repo} repository. Please install ${artifact} first if you want to use a non-published artifact." - exit 1 -} - -# gets the output folder where all sources and dependencies will be located. -get_output_folder() { - if [[ $(basename $(pwd)) != "output" ]]; then - echo "$(pwd)/output" - else - echo $(pwd) - fi -} - -detect_os_architecture() { - local os_type - local os_architecture - os_type=$(uname -sm) - case "${os_type}" in - *"Linux x86_64"*) - os_architecture="linux-x86_64" - ;; - *"Darwin x86_64"*) - os_architecture="osx-x86_64" - ;; - *) - os_architecture="osx-aarch_64" - ;; - esac - echo "${os_architecture}" -} - - -# copies $1 as a folder as $2 only if $1 exists -copy_directory_if_exists() { - local base_folder=$1 - local folder_prefix=$2 - local destination_folder=$3 - if [ ! -d "${base_folder}" ]; then - return - fi - pushd "${base_folder}" - if [[ $(find . -maxdepth 1 -type d -name "${folder_prefix}*" | wc -l ) -gt 0 ]]; then - cp -r ${base_folder}/${folder_prefix}* "${destination_folder}" - fi - popd # base_folder -} - -# computes proto_path from a given folder of GAPIC sources -# It will inspect the proto library to compute the path -get_proto_path_from_preprocessed_sources() { - set -e - local sources=$1 - pushd "${sources}" > /dev/null - local proto_library - proto_library=$(find . -maxdepth 1 -type d -name 'proto-*' | sed 's/\.\///') - local found_libraries - found_libraries=$(echo "${proto_library}" | wc -l) - if [[ -z ${proto_library} ]]; then - echo "no proto libraries found in the supplied sources path" - exit 1 - elif [ ${found_libraries} -gt 1 ]; then - echo "more than one proto library found in the supplied sources path" - echo "cannot decide for a service version" - exit 1 - fi - pushd "$(pwd)/${proto_library}/src/main/proto" > /dev/null - local result=$(find . -type f -name '*.proto' | head -n 1 | xargs dirname | sed 's/\.\///') - popd > /dev/null # proto_library - popd > /dev/null # sources - echo "${result}" -} - -# for a pre-processed library stored in $preprocessed_sources_path, a folder -# tree is built on $target_folder so it looks like a googleapis-gen folder and -# is therefore consumable by OwlBot CLI -build_owlbot_cli_source_folder() { - local postprocessing_target=$1 - local target_folder=$2 - local preprocessed_sources_path=$3 - local proto_path=$4 - if [[ -z "${proto_path}" ]]; then - proto_path=$(get_proto_path_from_preprocessed_sources "${preprocessed_sources_path}") - fi - owlbot_staging_folder="${postprocessing_target}/owl-bot-staging" - mkdir -p "${owlbot_staging_folder}" - - # By default (thanks to generation templates), .OwlBot-hermetic.yaml `deep-copy` section - # references a wildcard pattern matching a folder - # ending with `-java` at the leaf of proto_path. We then use a generated-java - # folder that will be picked up by copy-code - mkdir -p "${target_folder}/${proto_path}/generated-java" - copy_directory_if_exists "${preprocessed_sources_path}" "proto" \ - "${target_folder}/${proto_path}/generated-java/proto-google-cloud-library" - copy_directory_if_exists "${preprocessed_sources_path}" "grpc" \ - "${target_folder}/${proto_path}/generated-java/grpc-google-cloud-library" - copy_directory_if_exists "${preprocessed_sources_path}" "gapic" \ - "${target_folder}/${proto_path}/generated-java/gapic-google-cloud-library" - copy_directory_if_exists "${preprocessed_sources_path}" "samples" \ - "${target_folder}/${proto_path}/generated-java/samples" - pushd "${target_folder}" - # create an empty commit so owl-bot-copy can process this as a repo - # (it cannot process non-git-repositories) - git init - git commit --allow-empty -m 'empty commit' - popd # target_folder -} - -# Convenience function to clone only the necessary folders from a git repository -sparse_clone() { - repo_url=$1 - paths=$2 - commitish=$3 - clone_dir=$(basename "${repo_url%.*}") - rm -rf "${clone_dir}" - git clone -n --depth=1 --no-single-branch --filter=tree:0 "${repo_url}" - pushd "${clone_dir}" - if [ -n "${commitish}" ]; then - git checkout "${commitish}" - fi - git sparse-checkout set --no-cone ${paths} - git checkout - popd -} - -# calls a function in utilities.py. THe first argument is the function name, the -# rest of the arguments are the positional arguments to such function -py_util() { - python3 "${utilities_script_dir}/utilities.py" "$@" -} - -get_gapic_generator_location() { - local generator_location - generator_location="${HOME}/.library_generation/gapic-generator-java.jar" - if [[ -n "${GAPIC_GENERATOR_LOCATION}" ]]; then - echo "${GAPIC_GENERATOR_LOCATION}" - elif [[ -f "${generator_location}" ]]; then - echo "${generator_location}" - else - echo "Can't find GAPIC generator in ${generator_location}." - exit 1 - fi -} - -get_protoc_location() { - local protoc_location - protoc_location="${HOME}/.library_generation/bin" - if [[ -n "${DOCKER_PROTOC_LOCATION}" ]]; then - echo "${DOCKER_PROTOC_LOCATION}" - elif [[ -d "${protoc_location}" ]]; then - echo "${protoc_location}" - else - echo "Can't find protoc in ${protoc_location}." - exit 1 - fi -} - -get_grpc_plugin_location() { - local grpc_location - grpc_location="${HOME}/.library_generation/protoc-gen-grpc-java.exe" - if [[ -n "${DOCKER_GRPC_LOCATION}" ]]; then - echo "${DOCKER_GRPC_LOCATION}" - elif [[ -f "${grpc_location}" ]]; then - echo "${grpc_location}" - else - echo "Can't find grpc plugin in ${grpc_location}." - exit 1 - fi -} - -get_java_formatter_location() { - local formatter_location - formatter_location="${HOME}/.library_generation/google-java-format.jar" - if [[ -n "${JAVA_FORMATTER_LOCATION}" ]]; then - echo "${JAVA_FORMATTER_LOCATION}" - elif [[ -f "${formatter_location}" ]]; then - echo "${formatter_location}" - else - echo "Can't find Java formatter in ${formatter_location}." - exit 1 - fi -} - -error_if_not_exists() { - local required_tool=$1 - if [[ ! -f "${required_tool}" ]]; then - >&2 echo "File ${required_tool} not found in the filesystem. " - >&2 echo "Please configure your environment and store the " - >&2 echo "required tools in this location." - exit 1 - fi -} - -# Backups existing Version.java files in the specified target directory -# Arguments: -# 1 - target_directory: The directory where to search for Version.java -# 2 - backup_directory: The directory where the backups will be stored -backup_version_java() { - local target_directory=$1 - local backup_directory=$2 - - pushd "${target_directory}" > /dev/null - find . -type f -path "*/src/*/stub/Version.java" -print0 | while IFS= read -r -d '' file; do - local backup_path="${backup_directory}/${file#./}" - mkdir -p "$(dirname "${backup_path}")" - cp "$file" "${backup_path}" - done - popd > /dev/null -} - -# Restores Version.java files from the backup directory to the target directory -# Arguments: -# 1 - target_directory: The directory where to restore Version.java -# 2 - backup_directory: The directory where the backups are stored -restore_version_java() { - local target_directory=$1 - local backup_directory=$2 - - if [[ ! -d "${backup_directory}" ]]; then - return 0 - fi - - pushd "${backup_directory}" > /dev/null - find . -type f -print0 | while IFS= read -r -d '' file; do - local relative_path="${file#./}" - local target_path="${target_directory}/${relative_path}" - mkdir -p "$(dirname "${target_path}")" - mv "$file" "${target_path}" - done - popd > /dev/null -} - -normalize_owlbot_yaml() { - local input_file="$1" - local output_file="$2" - local library_name="$3" - - # Step 1: Remove library name prefix and normalize indentation for matching lines. - # Step 2: Normalize indentation for lines that didn't have the prefix. - # Using | as sed delimiter to avoid escaping forward slashes. - sed -e "s|^\s*- \"/${library_name}|- \"|" \ - -e "s|^\s*- \"|- \"|" \ - "${input_file}" > "${output_file}" -} diff --git a/sdk-platform-java/hermetic_build/release_note_generation/__init__.py b/sdk-platform-java/hermetic_build/release_note_generation/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/release_note_generation/cli/generate_release_note.py b/sdk-platform-java/hermetic_build/release_note_generation/cli/generate_release_note.py deleted file mode 100644 index 2dd35ebf826e..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/cli/generate_release_note.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from typing import Optional -import click as click -from release_note_generation.generate_pr_description import generate_pr_descriptions -from common.model.generation_config import GenerationConfig -from common.utils.generation_config_comparator import compare_config - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--baseline-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - This config file is used for commit history generation, not library - generation. - """, -) -@click.option( - "--current-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml that contains the - metadata about library generation. - """, -) -@click.option( - "--repository-path", - type=str, - default=".", - show_default=True, - help=""" - The path where the file `pr_description.txt`, which contains the release - notes, will be sent. - If not specified, the file will be generated to the current working - directory. - """, -) -def generate( - baseline_generation_config_path: Optional[str], - current_generation_config_path: Optional[str], - repository_path: str, -) -> None: - if ( - baseline_generation_config_path is None - or current_generation_config_path is None - ): - print( - "One of the generation configs is not specified, do not generate " - "the description." - ) - return - baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) - current_generation_config_path = os.path.abspath(current_generation_config_path) - if not ( - os.path.isfile(baseline_generation_config_path) - and os.path.isfile(current_generation_config_path) - ): - print( - "One of the generation configs does not exist, do not generate " - "the description." - ) - return - config_change = compare_config( - baseline_config=GenerationConfig.from_yaml(baseline_generation_config_path), - current_config=GenerationConfig.from_yaml(current_generation_config_path), - ) - generate_pr_descriptions( - config_change=config_change, - description_path=os.path.abspath(repository_path), - ) - - -if __name__ == "__main__": - main() diff --git a/sdk-platform-java/hermetic_build/release_note_generation/commit_message_formatter.py b/sdk-platform-java/hermetic_build/release_note_generation/commit_message_formatter.py deleted file mode 100644 index c86c1df1920c..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/commit_message_formatter.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re -from git import Commit - -from common.model.config_change import ConfigChange, ChangeType -from common.model.generation_config import ( - GAPIC_GENERATOR_VERSION, - LIBRARIES_BOM_VERSION, -) - -PARAM_TO_COMMIT_MESSAGE = { - GAPIC_GENERATOR_VERSION: "fix(deps): update the Java code generator (gapic-generator-java) to", - LIBRARIES_BOM_VERSION: "chore: update the libraries_bom version to", -} - - -def format_commit_message(commits: dict[Commit, str], is_monorepo: bool) -> list[str]: - """ - Format commit messages. Add library_name to conventional commit messages if - is_monorepo is True; otherwise no op. - - :param commits: a mapping from commit to library_name. - :param is_monorepo: whether it's monorepo or not. - :return: formatted commit messages. - """ - all_commits = [] - # please see go/java-client-releasing#conventional-commit-messages - # for conventional commit. - type_regex = re.compile(r"(feat|fix|docs|deps|test|samples|chore)!?:.*") - for commit, library_name in commits.items(): - # a commit message may contain multiple lines, we need to - # add library_name for each line. - messages = [] - for message_line in commit.message.split("\n"): - # add library name to a conventional commit message; - # otherwise no op. - if type_regex.search(message_line): - commit_type, _, summary = message_line.partition(":") - formatted_message = ( - f"{commit_type}: [{library_name}]{str(summary).rstrip()}" - if is_monorepo - else f"{commit_type}:{str(summary).rstrip()}" - ) - messages.append(formatted_message) - else: - messages.append(message_line) - all_commits.extend(wrap_googleapis_commit(commit, messages)) - return all_commits - - -def format_repo_level_change(config_change: ConfigChange) -> list[str]: - """ - Format commit messages regarding repo-level changes. - - :param config_change: - :return: commit messages regarding repo-level changes. - """ - messages = [] - for repo_level_change in config_change.change_to_libraries.get( - ChangeType.REPO_LEVEL_CHANGE, [] - ): - message = f"chore: update repo-level parameter {repo_level_change.changed_param} to {repo_level_change.current_value}" - if repo_level_change.changed_param in PARAM_TO_COMMIT_MESSAGE: - message = f"{PARAM_TO_COMMIT_MESSAGE.get(repo_level_change.changed_param)} {repo_level_change.current_value}" - messages.extend(__wrap_nested_commit([message])) - return messages - - -def wrap_googleapis_commit(commit: Commit, messages: list[str]) -> list[str]: - """ - Wrap message between `BEGIN_NESTED_COMMIT` and `BEGIN_NESTED_COMMIT`. - - :param commit: a GitHub commit. - :param messages: a (multi-line) commit message, one line per item. - :return: wrapped messages. - """ - messages.append(f"Source Link: {commit_link(commit)}") - return __wrap_nested_commit(messages) - - -def wrap_override_commit(messages: list[str]) -> list[str]: - """ - Wrap message between `BEGIN_COMMIT_OVERRIDE` and `END_COMMIT_OVERRIDE`. - - :param messages: a (multi-line) commit message, one line per item. - :return: wrapped messages. - """ - result = ["BEGIN_COMMIT_OVERRIDE"] - result.extend(messages) - result.append("END_COMMIT_OVERRIDE") - return result - - -def commit_link(commit: Commit) -> str: - """ - Create a link to the commit in Markdown format. - - :param commit: a GitHub commit. - :return: a link in Markdown format. - """ - short_sha = commit.hexsha[:7] - return f"[googleapis/googleapis@{short_sha}](https://github.com/googleapis/googleapis/commit/{commit.hexsha})" - - -def __wrap_nested_commit(messages: list[str]) -> list[str]: - """ - Wrap message between `BEGIN_NESTED_COMMIT` and `BEGIN_NESTED_COMMIT`. - - :param messages: a (multi-line) commit message, one line per item. - :return: wrapped messages. - """ - result = ["BEGIN_NESTED_COMMIT"] - result.extend(messages) - result.append("END_NESTED_COMMIT") - return result diff --git a/sdk-platform-java/hermetic_build/release_note_generation/generate_pr_description.py b/sdk-platform-java/hermetic_build/release_note_generation/generate_pr_description.py deleted file mode 100755 index f47985ad82b0..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/generate_pr_description.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import calendar -import os -import shutil -from typing import Dict -from git import Commit, Repo - -from common.model.config_change import ConfigChange -from common.utils.proto_path_utils import find_versioned_proto_path -from release_note_generation.commit_message_formatter import ( - format_commit_message, - format_repo_level_change, - commit_link, - wrap_override_commit, -) - -EMPTY_MESSAGE = "" - - -def generate_pr_descriptions( - config_change: ConfigChange, - description_path: str, - repo_url: str = "https://github.com/googleapis/googleapis.git", -) -> None: - """ - Generate pull request description from configuration comparison result. - - The pull request description contains repo-level changes, if applicable, - and googleapis commit from baseline config (exclusive) to current config - (inclusive). - - The pull request description will be generated into - description_path/pr_description.txt. - - No pr_description.txt will be generated if no changes in the configurations. - - :param config_change: a ConfigChange object, containing changes in baseline - and current generation configurations. - :param description_path: the path to which the pull request description - file goes. - :param repo_url: the GitHub repository from which retrieves the commit - history. - """ - repo_level_message = format_repo_level_change(config_change) - paths = config_change.current_config.get_proto_path_to_library_name() - description = get_repo_level_commit_messages( - repo_url=repo_url, - current_commit_sha=config_change.current_config.googleapis_commitish, - baseline_commit_sha=config_change.baseline_config.googleapis_commitish, - paths=paths, - is_monorepo=config_change.current_config.is_monorepo(), - repo_level_message=repo_level_message, - ) - - if description == EMPTY_MESSAGE: - print("Empty commit messages, skip creating pull request description.") - return - - description_file = f"{description_path}/pr_description.txt" - print(f"Writing pull request description to {description_file}") - with open(description_file, "w+") as f: - f.write(description) - - -def get_repo_level_commit_messages( - repo_url: str, - current_commit_sha: str, - baseline_commit_sha: str, - paths: Dict[str, str], - is_monorepo: bool, - repo_level_message: list[str], -) -> str: - """ - Combine commit messages of a repository from latest_commit to - baseline_commit. Only commits which change files in a pre-defined - file paths will be considered. - Note that baseline_commit should be an ancestor of or the same as - latest_commit. - - :param repo_url: the url of the repository. - :param current_commit_sha: the newest commit to be considered in - selecting commit message. - :param baseline_commit_sha: the oldest commit to be considered in - selecting commit message. This commit should be an ancestor of - :param paths: a mapping from file paths to library_name. - :param is_monorepo: whether to generate commit messages in a monorepo. - :param repo_level_message: commit messages regarding repo-level changes. - :return: commit messages. - :raise ValueError: if current_commit is older than or equal to - baseline_commit. - """ - tmp_dir = "/tmp/repo" - shutil.rmtree(tmp_dir, ignore_errors=True) - os.mkdir(tmp_dir) - repo = Repo.clone_from(repo_url, tmp_dir) - current_commit = repo.commit(current_commit_sha) - baseline_commit = repo.commit(baseline_commit_sha) - current_commit_time = __get_commit_timestamp(current_commit) - baseline_commit_time = __get_commit_timestamp(baseline_commit) - if current_commit_time < baseline_commit_time: - raise ValueError( - f"current_commit ({current_commit_sha[:7]}, committed on " - f"{current_commit_time}) should be newer than or equal to " - f"baseline_commit ({baseline_commit_sha[:7]}, committed on " - f"{baseline_commit_time}) (is this branch up to date with " - f"the base branch?)." - ) - qualified_commits = {} - commit = current_commit - while str(commit.hexsha) != baseline_commit_sha: - commit_and_name = __filter_qualified_commit(paths=paths, commit=commit) - if commit_and_name != (): - qualified_commits[commit_and_name[0]] = commit_and_name[1] - commit_parents = commit.parents - if len(commit_parents) == 0: - break - commit = commit_parents[0] - shutil.rmtree(tmp_dir, ignore_errors=True) - - return __combine_commit_messages( - current_commit=current_commit, - baseline_commit=baseline_commit, - commits=qualified_commits, - is_monorepo=is_monorepo, - repo_level_message=repo_level_message, - ) - - -def __filter_qualified_commit(paths: Dict[str, str], commit: Commit) -> (Commit, str): - """ - Returns a tuple of a commit and libray_name. - A qualified commit means at least one file, excluding BUILD.bazel, changes - in that commit is within the versioned proto_path in paths. - - :param paths: a mapping from versioned proto_path to library_name. - :param commit: a commit under consideration. - :return: a tuple of a commit and library_name if the commit is - qualified; otherwise an empty tuple. - """ - for file in commit.stats.files.keys(): - versioned_proto_path = find_versioned_proto_path(file) - if versioned_proto_path in paths and (not file.endswith("BUILD.bazel")): - return commit, paths[versioned_proto_path] - return () - - -def __combine_commit_messages( - current_commit: Commit, - baseline_commit: Commit, - commits: Dict[Commit, str], - is_monorepo: bool, - repo_level_message: list[str], -) -> str: - description = [] - if current_commit != baseline_commit: - description.append( - f"This pull request is generated with proto changes between " - f"{commit_link(baseline_commit)} (exclusive) " - f"and {commit_link(current_commit)} (inclusive).\n", - ) - commit_message = repo_level_message - commit_message.extend( - format_commit_message(commits=commits, is_monorepo=is_monorepo) - ) - if len(commit_message) == 0: - return EMPTY_MESSAGE - description.extend(wrap_override_commit(commit_message)) - return "\n".join(description) - - -def __get_commit_timestamp(commit: Commit) -> int: - """ - # Convert datetime to UTC timestamp. For more info: - # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime - - :param commit: a Commit object - :return: the timestamp of the commit - """ - return calendar.timegm(commit.committed_datetime.utctimetuple()) diff --git a/sdk-platform-java/hermetic_build/release_note_generation/requirements.in b/sdk-platform-java/hermetic_build/release_note_generation/requirements.in deleted file mode 100644 index 4c9d8e47d763..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -click==8.1.7 -GitPython==3.1.43 \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/release_note_generation/requirements.txt b/sdk-platform-java/hermetic_build/release_note_generation/requirements.txt deleted file mode 100644 index 66fc4abe33c8..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --generate-hashes hermetic_build/release_note_generation/requirements.in -# -click==8.1.7 \ - --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ - --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de - # via -r hermetic_build/release_note_generation/requirements.in -gitdb==4.0.11 \ - --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ - --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b - # via gitpython -GitPython==3.1.44 \ - --hash=sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110 \ - --hash=sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269 - # via -r hermetic_build/release_note_generation/requirements.in -smmap==5.0.2 \ - --hash=sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5 \ - --hash=sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e - # via gitdb diff --git a/sdk-platform-java/hermetic_build/release_note_generation/setup.py b/sdk-platform-java/hermetic_build/release_note_generation/setup.py deleted file mode 100755 index 71728c7ef253..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/setup.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Package information of library_generation python scripts -""" - -from setuptools import setup - -setup( - name="release_note_generation", - version="0.1", - python_requires=">=3.12", - package_dir={ - "release_note_generation": ".", - }, -) diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/__init__.py b/sdk-platform-java/hermetic_build/release_note_generation/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/cli/__init__.py b/sdk-platform-java/hermetic_build/release_note_generation/tests/cli/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py b/sdk-platform-java/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py deleted file mode 100644 index e1ee1e31e8a7..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from click.testing import CliRunner -from release_note_generation.cli.generate_release_note import generate - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resource_dir = os.path.join(script_dir, "..", "resources", "test_generate_release_note") - - -class GenerateReleaseNoteTest(unittest.TestCase): - def test_gen_release_note_with_no_baseline_config_does_not_generate_note(self): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke(generate) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - - def test_gen_release_note_with_no_current_config_does_not_generate_note(self): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, ["--baseline-generation-config-path=any_config"] - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - - def test_gen_release_note_with_nonexistent_baseline_config_does_not_generate_note( - self, - ): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, - [ - "--baseline-generation-config-path=non_existent_config", - "--current-generation-config-path=not_relevant", - ], - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - - def test_gen_release_note_with_nonexistent_current_config_does_not_generate_note( - self, - ): - cwd = os.getcwd() - os.chdir(resource_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, - [ - "--baseline-generation-config-path=empty_gen_config.yaml", - "--current-generation-config-path=non_existent_config", - ], - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - os.chdir(cwd) diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py b/sdk-platform-java/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py deleted file mode 100644 index b7891495eede..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest -from unittest.mock import patch -from common.model.config_change import ( - ConfigChange, - ChangeType, - LibraryChange, -) -from common.model.generation_config import GenerationConfig -from release_note_generation.commit_message_formatter import ( - format_commit_message, - commit_link, - format_repo_level_change, - wrap_googleapis_commit, - wrap_override_commit, -) - -gen_config = GenerationConfig( - gapic_generator_version="1.2.3", googleapis_commitish="123abc", libraries=[] -) - - -class CommitMessageFormatterTest(unittest.TestCase): - def test_format_commit_message_should_add_library_name_for_conventional_commit( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: [example_library] a commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, True), - ) - - def test_format_commit_message_should_add_library_name_for_mutliline_conventional_commit( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nfix: an another commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: [example_library] a commit message", - "fix: [example_library] an another commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, True), - ) - - def test_format_commit_message_should_not_add_library_name_for_nonconvnentional_commit( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "PiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, True), - ) - - def test_format_commit_message_should_not_add_library_name_if_not_monorepo(self): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: a commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, False), - ) - - def test_format_commit_message_should_not_add_library_name_for_multiline_commit_if_not_monorepo( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nfix: an another commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: a commit message", - "fix: an another commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, False), - ) - - def test_wrap_nested_commit_success(self): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.hexsha = "1234567abcdefg" - messages = ["a commit message", "another message"] - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "a commit message", - "another message", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - wrap_googleapis_commit(commit, messages), - ) - - def test_wrap_override_commit_success(self): - messages = ["a commit message", "another message"] - self.assertEqual( - [ - "BEGIN_COMMIT_OVERRIDE", - "a commit message", - "another message", - "END_COMMIT_OVERRIDE", - ], - wrap_override_commit(messages), - ) - - def test_commit_link_success(self): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.hexsha = "1234567abcdefg" - self.assertEqual( - "[googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - commit_link(commit), - ) - - def test_format_repo_level_change_success(self): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", current_value="1.2.3" - ), - LibraryChange( - changed_param="libraries_bom_version", current_value="2.3.4" - ), - LibraryChange( - changed_param="protoc_version", current_value="3.4.5" - ), - ] - }, - baseline_config=gen_config, - current_config=gen_config, - ) - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3", - "END_NESTED_COMMIT", - "BEGIN_NESTED_COMMIT", - "chore: update the libraries_bom version to 2.3.4", - "END_NESTED_COMMIT", - "BEGIN_NESTED_COMMIT", - "chore: update repo-level parameter protoc_version to 3.4.5", - "END_NESTED_COMMIT", - ], - format_repo_level_change(config_change), - ) diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py b/sdk-platform-java/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py deleted file mode 100644 index 9659af5b2d71..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from filecmp import cmp -from release_note_generation.generate_pr_description import ( - get_repo_level_commit_messages, - generate_pr_descriptions, -) -from common.model.config_change import ( - ConfigChange, - ChangeType, - LibraryChange, -) -from common.model.gapic_config import GapicConfig -from common.model.generation_config import GenerationConfig -from common.model.library_config import LibraryConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "resources", "goldens") - - -class GeneratePrDescriptionTest(unittest.TestCase): - def test_get_commit_messages_current_is_older_raise_exception(self): - # committed on April 1st, 2024 - current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" - # committed on April 2nd, 2024 - baseline_commit = "d5020fff4cbe108bdf506074791c56cff7840bef" - self.assertRaisesRegex( - ValueError, - "newer than", - get_repo_level_commit_messages, - "https://github.com/googleapis/googleapis.git", - current_commit, - baseline_commit, - {}, - True, - [], - ) - - def test_get_commit_messages_with_same_current_and_baseline_returns_empty_message( - self, - ): - # committed on April 1st, 2024 - current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" - baseline_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" - self.assertEqual( - "", - get_repo_level_commit_messages( - "https://github.com/googleapis/googleapis.git", - current_commit, - baseline_commit, - {}, - True, - [], - ), - ) - - def test_generate_pr_description_with_no_change_in_config(self): - commit_sha = "36441693dddaf0ed73951ad3a15c215a332756aa" - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish=commit_sha, - libraries_bom_version="", - # use empty libraries to make sure no qualified commit between - # two commit sha. - libraries=[], - ) - pr_description_path = os.path.join(os.getcwd(), "no_config_change") - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={}, - baseline_config=config, - current_config=config, - ), - description_path=pr_description_path, - ) - self.assertFalse(os.path.isfile(f"{pr_description_path}/pr_description.txt")) - - def test_generate_pr_description_does_not_create_pr_description_without_qualified_commit( - self, - ): - # committed on May 22nd, 2024 - old_commit_sha = "30717c0b0c9966906880703208a4c820411565c4" - # committed on May 23rd, 2024 - new_commit_sha = "eeed69d446a90eb4a4a2d1762c49d637075390c1" - pr_description_path = os.path.join(os.getcwd(), "no_qualified_commit") - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={}, - baseline_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=old_commit_sha, - # use empty libraries to make sure no qualified commit between - # two commit sha. - libraries=[], - ), - current_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=new_commit_sha, - # use empty libraries to make sure no qualified commit between - # two commit sha. - libraries=[], - ), - ), - description_path=pr_description_path, - ) - self.assertFalse(os.path.isfile(f"{pr_description_path}/pr_description.txt")) - - def test_generate_pr_description_with_combined_message( - self, - ): - # no other commits between these two commits. - baseline_commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" - documentai_commit_sha = "0cea7170404bec3d994f43db4fa292f5034cbe9a" - cwd = os.getcwd() - library = LibraryConfig( - api_shortname="documentai", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[GapicConfig(proto_path="google/cloud/documentai/v1")], - ) - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", - current_value="1.2.3", - ), - LibraryChange( - changed_param="libraries_bom_version", current_value="2.3.4" - ), - ], - ChangeType.GOOGLEAPIS_COMMIT: [], - }, - baseline_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=baseline_commit_sha, - libraries=[library], - ), - current_config=GenerationConfig( - gapic_generator_version="1.2.3", - googleapis_commitish=documentai_commit_sha, - libraries_bom_version="2.3.4", - libraries=[library], - ), - ), - description_path=cwd, - ) - self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) - self.assertTrue( - cmp( - f"{resources_dir}/pr_description-golden.txt", - f"{cwd}/pr_description.txt", - ), - "The generated PR description does not match the expected golden file", - ) - os.remove(f"{cwd}/pr_description.txt") - - def test_generate_pr_description_with_repo_level_change_without_qualified_commit( - self, - ): - # no other commits between these two commits. - baseline_commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" - current_commit_sha = "0cea7170404bec3d994f43db4fa292f5034cbe9a" - cwd = os.getcwd() - library = LibraryConfig( - api_shortname="example_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[GapicConfig(proto_path="google/example/v1")], - ) - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", - current_value="1.2.3", - ), - LibraryChange( - changed_param="libraries_bom_version", current_value="2.3.4" - ), - ], - ChangeType.GOOGLEAPIS_COMMIT: [], - }, - baseline_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=baseline_commit_sha, - libraries=[library], - ), - current_config=GenerationConfig( - gapic_generator_version="1.2.3", - googleapis_commitish=current_commit_sha, - libraries_bom_version="2.3.4", - libraries=[library], - ), - ), - description_path=cwd, - ) - self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) - self.assertTrue( - cmp( - f"{resources_dir}/repo_level_and_no_qualified_commit_pr_description-golden.txt", - f"{cwd}/pr_description.txt", - ), - "The generated PR description does not match the expected golden file", - ) - os.remove(f"{cwd}/pr_description.txt") - - def test_generate_pr_description_create_description_with_only_repo_level_change( - self, - ): - commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" - cwd = os.getcwd() - library = LibraryConfig( - api_shortname="documentai", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[GapicConfig(proto_path="google/cloud/documentai/v1")], - ) - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", - current_value="1.2.3", - ) - ], - ChangeType.GOOGLEAPIS_COMMIT: [], - }, - baseline_config=GenerationConfig( - gapic_generator_version="1.2.2", - googleapis_commitish=commit_sha, - libraries=[library], - ), - current_config=GenerationConfig( - gapic_generator_version="1.2.3", - googleapis_commitish=commit_sha, - libraries=[library], - ), - ), - description_path=cwd, - ) - self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) - self.assertTrue( - cmp( - f"{resources_dir}/repo_level_only_pr_description-golden.txt", - f"{cwd}/pr_description.txt", - ), - "The generated PR description does not match the expected golden file", - ) - os.remove(f"{cwd}/pr_description.txt") diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/pr_description-golden.txt b/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/pr_description-golden.txt deleted file mode 100644 index 1a0f87493642..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/pr_description-golden.txt +++ /dev/null @@ -1,17 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update the libraries_bom version to 2.3.4 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: Make Layout Parser generally available in V1 - -PiperOrigin-RevId: 638924855 - -Source Link: [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt b/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt deleted file mode 100644 index 81f7c0c4d245..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt +++ /dev/null @@ -1,10 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update the libraries_bom version to 2.3.4 -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt b/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt deleted file mode 100644 index f89901f8c152..000000000000 --- a/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt +++ /dev/null @@ -1,5 +0,0 @@ -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml b/sdk-platform-java/hermetic_build/release_note_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/sdk-platform-java/release-please-config.json b/sdk-platform-java/release-please-config.json index db518b5db8b6..035230bfbbd8 100644 --- a/sdk-platform-java/release-please-config.json +++ b/sdk-platform-java/release-please-config.json @@ -11,9 +11,6 @@ ".cloudbuild/graalvm/cloudbuild-test-a.yaml", ".cloudbuild/graalvm/cloudbuild-test-b.yaml", ".cloudbuild/graalvm/cloudbuild-test-c.yaml", - ".cloudbuild/library_generation/cloudbuild-library-generation-push.yaml", - ".cloudbuild/library_generation/library_generation.Dockerfile", - ".cloudbuild/library_generation/library_generation_airlock.Dockerfile", "hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-native-a.cfg", "hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-native-b.cfg", "hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-native-c.cfg",