From 746c22826aacb53d4134de8b46e6834bcbc5fb00 Mon Sep 17 00:00:00 2001 From: Min Zhu Date: Tue, 30 Jun 2026 14:41:48 -0400 Subject: [PATCH] chore: remove hermetic build toolset and workflow for new library --- .../new-client-library-generator/SKILL.md | 124 -------- .../references/generation_config_schema.md | 57 ---- .../references/generation_guide.md | 75 ----- .../references/service_yaml_mapping.md | 41 --- .../generate_new_client_hermetic_build.yaml | 117 -------- .../new_client_hermetic_build/README.md | 266 ----------------- .../add-new-client-config.py | 278 ------------------ .../generate-arguments.py | 43 --- .../new_client_hermetic_build/requirements.in | 3 - .../requirements.txt | 66 ----- 10 files changed, 1070 deletions(-) delete mode 100644 .agents/skills/new-client-library-generator/SKILL.md delete mode 100644 .agents/skills/new-client-library-generator/references/generation_config_schema.md delete mode 100644 .agents/skills/new-client-library-generator/references/generation_guide.md delete mode 100644 .agents/skills/new-client-library-generator/references/service_yaml_mapping.md delete mode 100644 .github/workflows/generate_new_client_hermetic_build.yaml delete mode 100644 generation/new_client_hermetic_build/README.md delete mode 100644 generation/new_client_hermetic_build/add-new-client-config.py delete mode 100644 generation/new_client_hermetic_build/generate-arguments.py delete mode 100644 generation/new_client_hermetic_build/requirements.in delete mode 100644 generation/new_client_hermetic_build/requirements.txt diff --git a/.agents/skills/new-client-library-generator/SKILL.md b/.agents/skills/new-client-library-generator/SKILL.md deleted file mode 100644 index d333fdda1576..000000000000 --- a/.agents/skills/new-client-library-generator/SKILL.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -name: new-client-library-generator -description: Generates new Google Cloud Java client libraries by processing service information and updating the hermetic build configuration. Use this skill when tasked with onboarding a new service or version to the google-cloud-java repository. ---- - -# New Client Library Generator - -This skill automates the process of adding a new client library to the `google-cloud-java` repository using the hermetic build system. It retrieves service information from a Buganizer ticket (which links to a Service YAML file) and runs the configuration script to update `generation_config.yaml`. - -## Workflow - -### 1. Retrieve Service Information from Buganizer - -Use the available Buganizer MCP server to fetch the ticket content. The ticket will contain a link to a Service YAML file. Parse this YAML file to extract the fields below. - -> [!IMPORTANT] -> Not all fields in the Service YAML file may exist. You must ensure that all **Required** fields are identified and populated before executing the generation script. - -**Required:** -- `api_shortname`: Unique service identifier (e.g., `alloydb`). -- `name_pretty`: Human-friendly name (e.g., `AlloyDB API`). -- `proto_path`: Versioned path to protos (e.g., `google/cloud/alloydb/v1`). Must include the version component — root-level paths like `google/cloud/alloydb` are not supported. -- `product_docs`: Product documentation URL (must start with `https://`). -- `api_description`: First sentence of the service summary. - -**Optional:** -- `rest_docs`: REST reference documentation URL. -- `rpc_docs`: RPC/proto reference documentation URL. -- `library_name`: Override the default `java-` directory name. -- `distribution_name`: Override Maven coordinates (default: `com.google.cloud:google-cloud-`). - -For the field-to-flag mapping, see [references/service_yaml_mapping.md](references/service_yaml_mapping.md) (Service YAML fields → script flags). - -### 2. Check for Conflicts - -Before running the script, verify `api_shortname` is not already in use: - -```bash -grep "api_shortname: " generation_config.yaml -``` - -If a conflict exists, determine a unique name or use `--library-name` to set a distinct directory. See [references/generation_guide.md](references/generation_guide.md) for examples. - -### 3. Special Cases - -Some APIs require non-default Maven coordinates and `api_shortname` values: - -| Proto path prefix | `--api-shortname` | `--distribution-name` | -|---------------------|------------------------------|----------------------------------------------------------| -| `google/maps/*` | `maps-` | `com.google.maps:google-maps-` | -| `google/shopping/*` | `shopping-` | `com.google.shopping:google-shopping-` | - -where `` is the value from the Service YAML. - -### 4. Execution - -#### Adding a new library - -If the library does not exist yet, run the script with the gathered information: - -```bash -python3 generation/new_client_hermetic_build/add-new-client-config.py add-new-library \ - --api-shortname="[API_SHORTNAME]" \ - --name-pretty="[NAME_PRETTY]" \ - --proto-path="[PROTO_PATH]" \ - --product-docs="[PRODUCT_DOCS]" \ - --api-description="[API_DESCRIPTION]" \ - [OPTIONAL_FLAGS] -``` - -The script modifies `generation_config.yaml` and sorts the `libraries` list alphabetically. - -To see all available flags: -```bash -python3 generation/new_client_hermetic_build/add-new-client-config.py add-new-library --help -``` - -#### Adding a new version to an existing library - -If the client library module already exists and the request is to add a new version, do NOT run the script. Instead, manually add the corresponding `proto_path` for the new version to the `GAPICs` list of the existing library entry in `generation_config.yaml`. - -Example entry update: -```yaml -- api_shortname: myapi - ... - GAPICs: - - proto_path: google/cloud/myapi/v1 - - proto_path: google/cloud/myapi/v2 # Manually added -``` - -### 5. Verification - -After execution or manual update: -1. Confirm `generation_config.yaml` has a new or updated entry with the correct fields. See [references/generation_config_schema.md](references/generation_config_schema.md). -2. Confirm `proto_path` under `GAPICs` includes a version component (e.g., `v1`, `v1beta`, `v1alpha`). -3. Confirm `product_documentation` starts with `https://`. - -### 6. Create a Branch and PR - -After verifying the changes: - -```bash -git checkout -b "new-library/[API_SHORTNAME]" -git add generation_config.yaml -git commit -m "feat: new module for [API_SHORTNAME]" -``` - -Then open a pull request. The Pull Request body must only contain the `add-new-library` command used. - -**PR Body Template:** - -```text -Command used: - -python3 generation/new_client_hermetic_build/add-new-client-config.py add-new-library \ - --api-shortname="[API_SHORTNAME]" \ - --name-pretty="[NAME_PRETTY]" \ - --proto-path="[PROTO_PATH]" \ - --product-docs="[PRODUCT_DOCS]" \ - --api-description="[API_DESCRIPTION]" \ - [OPTIONAL_FLAGS] -``` - -The hermetic library generation workflow will be triggered automatically upon changes to `generation_config.yaml`. diff --git a/.agents/skills/new-client-library-generator/references/generation_config_schema.md b/.agents/skills/new-client-library-generator/references/generation_config_schema.md deleted file mode 100644 index 075db9054032..000000000000 --- a/.agents/skills/new-client-library-generator/references/generation_config_schema.md +++ /dev/null @@ -1,57 +0,0 @@ -# generation_config.yaml Schema - -The `generation_config.yaml` file at the root of the repository controls the generation of all client libraries. - -## Key Fields - -- `gapic_generator_version`: The version of the GAPIC generator used globally (unless overridden). -- `googleapis_commitish`: The commit of the `googleapis` repository used as the source for protos. -- `libraries`: A list of library configurations. - -### Library Configuration Fields - -**Required (always present):** -- `api_shortname`: (String) Identifier (e.g., `alloydb`). -- `name_pretty`: (String) Display name (e.g., `AlloyDB API`). -- `product_documentation`: (URL) Product documentation URL. -- `api_description`: (String) Service description. -- `client_documentation`: (URL) Auto-generated link to the generated client docs. -- `release_level`: (String) Usually `preview` for new libraries. -- `distribution_name`: (String) Maven coordinates (e.g., `com.google.cloud:google-cloud-alloydb`). -- `api_id`: (String) API identifier (e.g., `alloydb.googleapis.com`). -- `library_type`: (String) Usually `GAPIC_AUTO`. -- `group_id`: (String) Maven group ID (e.g., `com.google.cloud`). -- `cloud_api`: (Boolean) `true` if distribution name starts with `google-cloud-`. -- `GAPICs`: (List) List of proto paths to generate from. - - `proto_path`: (String) Path to versioned protos (must include version, e.g., `google/cloud/alloydb/v1`). - -**Optional:** -- `library_name`: (String) Override the output directory name (without `java-` prefix). -- `rest_documentation`: (URL) REST reference documentation URL. -- `rpc_documentation`: (URL) RPC/proto reference documentation URL. -- `requires_billing`: (Boolean) Whether billing is required (default: `true`). -- `api_reference`: (URL) API reference documentation link. -- `codeowner_team`: (String) GitHub team responsible for this library. -- `googleapis_commitish`: (String) Pin to a specific `googleapis/googleapis` commit (overrides repo-level setting). -- `issue_tracker`: (URL) Issue tracker for this library. -- `extra_versioned_modules`: (String) Extra modules managed via `versions.txt`. -- `excluded_dependencies`: (String) Comma-separated dependencies excluded from postprocessing. -- `excluded_poms`: (String) Comma-separated pom files excluded from postprocessing. - -## Example Library Entry - -```yaml -- api_shortname: alloydb - name_pretty: AlloyDB API - product_documentation: https://cloud.google.com/alloydb/docs - api_description: AlloyDB for PostgreSQL is an open source-compatible database service. - client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-alloydb/latest/overview - release_level: preview - distribution_name: com.google.cloud:google-cloud-alloydb - api_id: alloydb.googleapis.com - library_type: GAPIC_AUTO - group_id: com.google.cloud - cloud_api: true - GAPICs: - - proto_path: google/cloud/alloydb/v1 -``` diff --git a/.agents/skills/new-client-library-generator/references/generation_guide.md b/.agents/skills/new-client-library-generator/references/generation_guide.md deleted file mode 100644 index 486c4c20253f..000000000000 --- a/.agents/skills/new-client-library-generator/references/generation_guide.md +++ /dev/null @@ -1,75 +0,0 @@ -# New Client Library Generation Guide - -This guide describes how to generate a new client library by: -1. Appending a new library configuration to `generation_config.yaml` using the provided scripts. -2. Relying on the automated GitHub Actions to generate the code once the configuration is pushed in a Pull Request. - -## Components - -### `generation/new_client_hermetic_build/add-new-client-config.py` -This script updates `generation_config.yaml` with the necessary metadata for a new library. It takes several arguments that map to fields in the configuration file. - -## Local Execution - -To add a new library configuration, you must run the `add-new-client-config.py` script. The parameters for this script are typically found in the **Service YAML** file linked in the Buganizer request. - -### Basic Usage - -```bash -python3 generation/new_client_hermetic_build/add-new-client-config.py add-new-library \ - --api-shortname="[API_SHORTNAME]" \ - --name-pretty="[NAME_PRETTY]" \ - --proto-path="[PROTO_PATH]" \ - --product-docs="[PRODUCT_DOCS]" \ - --api-description="[API_DESCRIPTION]" -``` - -### Parameter Details - -#### API Short Name (`--api-shortname`) -The unique identifier for the library (e.g., `alloydb`). This is used to determine the directory name and default artifact name. -- **Source:** `api_short_name` in the Service YAML. - -#### Proto Path (`--proto-path`) -The path from the root of the `googleapis` repository to the directory containing the versioned proto definitions. -- **Example:** `google/cloud/alloydb/v1beta` -- **Note:** Root-level paths like `google/cloud/alloydb` are not supported. - -#### Name Pretty (`--name-pretty`) -The human-friendly name of the API. -- **Source:** `title` in the Service YAML. -- **Example:** `AlloyDB API` - -#### Product Docs (`--product-docs`) -The URL for the product documentation. Must start with `https://`. -- **Source:** `documentation_uri` in the Service YAML. - -#### API Description (`--api-description`) -A concise summary of the API for the README. -- **Source:** `documentation.summary` or `documentation.overview` in the Service YAML. Use the first sentence. - -## Prerequisites - -Ensure you have the following set up in your local environment: - -1. **Python 3.9+**: The scripts require Python 3.9 or higher. -2. **Dependencies**: Install the required Python packages from the root of the repository: - ```bash - pip install -r generation/new_client_hermetic_build/requirements.txt - ``` - -## Advanced Options - -For full control over the generation (e.g., overriding Maven coordinates or library names), run the script with the `--help` flag: - -```bash -python3 generation/new_client_hermetic_build/add-new-client-config.py add-new-library --help -``` - -### Special Case: Non-Cloud APIs -Some libraries (like Google Maps or Shopping) require special handling for Maven coordinates. - -| API Path Prefix | `--api-shortname` | `--distribution-name` | -|---|---|---| -| `google/maps/*` | `maps-` | `com.google.maps:google-maps-` | -| `google/shopping/*` | `shopping-` | `com.google.shopping:google-shopping-` | diff --git a/.agents/skills/new-client-library-generator/references/service_yaml_mapping.md b/.agents/skills/new-client-library-generator/references/service_yaml_mapping.md deleted file mode 100644 index f865062aee11..000000000000 --- a/.agents/skills/new-client-library-generator/references/service_yaml_mapping.md +++ /dev/null @@ -1,41 +0,0 @@ -# Service YAML Mapping to Generation Script Parameters - -This document explains how to extract information from a Service YAML file (linked in a Buganizer ticket) to populate the parameters for `generation/new_client_hermetic_build/add-new-client-config.py`. - -| Service YAML Field | Script Parameter (`--flag`) | Description | -|---|---|---| -| `api_short_name` | `api-shortname` | Unique identifier for the service (e.g., `alloydb`). | -| (extracted from proto) | `proto-path` | Path from the root of the googleapis repository to the versioned proto directory (e.g., `google/cloud/alloydb/v1alpha`). Root-level paths like `google/cloud/alloydb` are not supported. | -| `title` | `name-pretty` | Human-friendly name (e.g., `AlloyDB API`). | -| `documentation_uri` | `product-docs` | Product documentation URL. Must start with `https://`. | -| `rest_reference_documentation_uri` | `rest-docs` | REST documentation URL. Optional. | -| `proto_reference_documentation_uri` | `rpc-docs` | RPC/Proto documentation URL. Optional. | -| `documentation.summary` or `documentation.overview` | `api-description` | Concise summary of the service. Use the first sentence. | - -## Example Mapping - -**Service YAML File excerpt:** -```yaml -title: "Discovery Engine API" -api_short_name: "discoveryengine" -documentation_uri: "https://cloud.google.com/generative-ai-app-builder/docs/introduction" -rest_reference_documentation_uri: "https://cloud.google.com/generative-ai-app-builder/docs/reference/rest" -proto_reference_documentation_uri: "https://cloud.google.com/generative-ai-app-builder/docs/reference/rpc" -documentation: - summary: "Discovery Engine for Search and Recommendations" -``` - -**Protos in `googleapis` repository:** -`google/cloud/discoveryengine/v1/` - -**Generated Command:** -```bash -python3 generation/new_client_hermetic_build/add-new-client-config.py add-new-library \ - --api-shortname="discoveryengine" \ - --name-pretty="Discovery Engine API" \ - --proto-path="google/cloud/discoveryengine/v1" \ - --product-docs="https://cloud.google.com/generative-ai-app-builder/docs/introduction" \ - --rest-docs="https://cloud.google.com/generative-ai-app-builder/docs/reference/rest" \ - --rpc-docs="https://cloud.google.com/generative-ai-app-builder/docs/reference/rpc" \ - --api-description="Discovery Engine for Search and Recommendations" -``` diff --git a/.github/workflows/generate_new_client_hermetic_build.yaml b/.github/workflows/generate_new_client_hermetic_build.yaml deleted file mode 100644 index e59fba01bd7b..000000000000 --- a/.github/workflows/generate_new_client_hermetic_build.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: Generate new GAPIC client library (Hermetic Build) -on: - workflow_dispatch: - # some inputs are omitted due to limit of 10 input arguments - inputs: - api_shortname: - required: true - type: string - description: "`api_shortname`: Name for the new directory name and (default) artifact name" - name_pretty: - required: true - type: string - description: "`name_pretty`: The human-friendly name that appears in README.md" - api_description: - required: true - description: "`api_description`: Description that appears in README.md" - proto_path: - required: true - type: string - description: | - `proto_path`: Path to proto file from the root of the googleapis repository to the - directory that contains the proto files (with the version). - For example, to generate `v2` of google/cloud/library, - you must pass google/cloud/library/v2 - product_docs: - required: true - type: string - description: "`product_docs`: Documentation URL that appears in README.md" - rest_docs: - required: false - type: string - description: | - `rest_docs`: If it exists, link to the REST Documentation for a service - rpc_docs: - required: false - type: string - description: | - `rpc_docs`: If it exists, link to the RPC Documentation for a service - library_name: - required: false - type: string - description: | - `library_name`: The directory name of the new library. By default it's - java-. **Do not include `java-` as it will be automatically prepended** - distribution_name: - required: false - type: string - description: | - `distribution_name`: Maven coordinates of the generated library. By default it's - com.google.cloud:google-cloud- - -jobs: - generate: - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.9' - cache: 'pip' # caching pip dependencies - - name: Install add-new-client-config.py dependencies - run: pip install --require-hashes -r generation/new_client_hermetic_build/requirements.txt - - name: Add entry to generation_config.yaml - id: config_generation - run: | - set -x - arguments=$(python generation/new_client_hermetic_build/generate-arguments.py) - echo "::set-output name=new_library_args::${arguments}" - echo "${arguments}" \ - | xargs python generation/new_client_hermetic_build/add-new-client-config.py add-new-library - env: - API_SHORTNAME: ${{ github.event.inputs.api_shortname }} - NAME_PRETTY: ${{ github.event.inputs.name_pretty }} - PROTO_PATH: ${{ github.event.inputs.proto_path }} - PRODUCT_DOCS: ${{ github.event.inputs.product_docs }} - REST_DOCS: ${{ github.event.inputs.rest_docs }} - RPC_DOCS: ${{ github.event.inputs.rpc_docs }} - API_DESCRIPTION: ${{ github.event.inputs.api_description }} - LIBRARY_NAME: ${{ github.event.inputs.library_name }} - DISTRIBUTION_NAME: ${{ github.event.inputs.distribution_name }} - - name: Push to branch and create PR - run: | - set -x - [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot" - - # create and push to branch in origin - # random_id allows multiple runs of this workflow - random_id=$(tr -dc A-Za-z0-9 [!IMPORTANT] -> `api_short_name` is not always unique across client libraries. -> In the instance that the `api_short_name` is already in use by an existing -> client library, you will need to determine a unique name OR to pass a unique -> `library_name`. -> See [Advanced Options](#advanced-options). - -### Proto path (`proto_path`) - -This is the path from the internal `google3/third_party/googleapis/stable` root -to the directory that contains the proto definitions for a specific version. -For example: `google/datastore/v2`. -Root-level proto paths like `google/datastore` are not supported. -Note that the internal `google3/third_party/googleapis/stable` directory is -mirrored externally in https://github.com/googleapis/googleapis/blob/master/. - -For example, if the Buganizer ticket includes: - -> Link to protos: `http://...(omit).../google/cloud/alloydb/v1alpha/alloydb_v1alpha.yaml`. - -then the corresponding external mirrored proto is here: `https://github.com/googleapis/googleapis/blob/master/google/cloud/alloydb/v1alpha/alloydb_v1alpha.yaml`. - -Therefore, the "proto path" value we supply to the command is -`google/cloud/alloydb/v1alpha`. - -We will publish a single module for a service that includes the specified version -(in the example, `v1alpha`). -Any future version must be manually added to the configuration yaml (`google-cloud-java/generation_config.yaml`) - -#### More than one `proto_path` - -If you need another `proto_path` in the library, you must manually add it -to `google-cloud-java/generation_config.yaml` after generating the new client. - -### Name pretty (`name_pretty`) - -The corresponding value in the Cloud Drop page is `title`. - -Example: `AlloyDB API` - -### Product Docs (`product_docs`) - -The corresponding value in the Cloud Drop page is `documentation_uri`. -The value must starts with "https://". - -Example: `https://cloud.google.com/alloydb/docs` - -### REST Docs (`rest_docs`) - -The corresponding value in the Cloud Drop page is `rest_reference_documentation_uri`. -The value must starts with "https://". - -Example: `https://cloud.google.com/alloydb/docs/reference/rest` - -If the value exists, add it as a flag to the python command below (see [Advanced -Options](#advanced-options): -`--rest-docs="https://cloud.google.com/alloydb/docs/reference/rest" \` - -### RPC Docs (`rpc_docs`) - -The corresponding value in the Cloud Drop page is `proto_reference_documentation_uri`. -The value must starts with "https://". - -Example: `https://cloud.google.com/speech-to-text/docs/reference/rpc` - -If the value exists, add it as a flag to the python command below (see [Advanced -Options](#advanced-options): -`--rpc-docs="https://cloud.google.com/speech-to-text/docs/reference/rpc" \` - -### API description (`api_description`) - -The corresponding value in the Cloud Drop page is `documentation.summary` or `documentation.overview`. -If both of those fields are missing, take the description from the product page -above. -Use the first sentence to keep it concise. - -Example: -``` - AlloyDB for PostgreSQL is an open source-compatible database service that - provides a powerful option for migrating, modernizing, or building - commercial-grade applications. - ``` - -### Distribution Name (`distribution_name`) - -This variable determines the Maven coordinates of the generated library. -It defaults to `com.google.cloud:google-cloud-{api_shortname}`. -This mainly affect the values in the generated `pom.xml` files. - -### Library Name (`library_name`) - -This variable indicates the output folder of the library. -For example, you can have two libraries with `alloydb` -(AlloyDB and AlloyDB Connectors) as `api_shortname`. -In order to avoid both libraries going to the default `java-alloydb` folder, -we can override this behavior by specifying a value like `alloydb-connectors` -so the AlloyDB Connectors goes to `java-alloydb-connectors`. - -## Prerequisites (for local environment) - -This section is only needed for the first _local_ run of this script. - -### Checkout google-cloud-java repository - -``` -$ git clone https://github.com/googleapis/google-cloud-java -$ git checkout main -$ git pull -``` - -### Install pyenv - -Install pyenv - -``` -curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer \ -| bash -``` - -Append the following lines to `$HOME/.bashrc`. - -``` -export PYENV_ROOT="$HOME/.pyenv" -command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" -eval "$(pyenv init -)" -eval "$(pyenv virtualenv-init -)" -``` - -Logout the shell and login again. You should be at the home directory. - -Assuming you have the following folder structure: -``` -~ (Home) - -> IdeaProjects/ - -> google-cloud-java - -> ... -``` -You can run these next commands in the home directory (or IdeaProjects). Otherwise, run it at `google-cloud-java`'s parent directory. - -Confirm pyenv installation succeeded: - -``` -~$ pyenv -pyenv 2.3.4 -Usage: pyenv [] - -Some useful pyenv commands are: - activate Activate virtual environment - commands List all available pyenv commands - deactivate Deactivate virtual environment -... -``` - -### Install Python 3.9 via pyenv - -``` -~$ pyenv install 3.9.13 -Downloading Python-3.9.13.tar.xz... --> https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tar.xz -Installing Python-3.9.13... -WARNING: The Python sqlite3 extension was not compiled. Missing the SQLite3 lib? -Installed Python-3.9.13 to /usr/local/google/home/suztomo/.pyenv/versions/3.9.13 -``` - -### Install Python v3.9.13 locally - -Run this command - -``` -$ pyenv local 3.9.13 -``` - -Confirm `python3.9` is installed: -``` -$ which python3.9 -/usr/local/google/home/suztomo/.pyenv/shims/python3.9 -``` - -### Install Python packages - -At the root of google-cloud-java repository clone, run: - -``` -$ python3.9 -m pip install -r generation/new_client_hermetic_build/requirements.txt -``` - -## Advanced Options - -In case the steps above don't show you how to specify the desired options, you -can run the `add-new-client-config.py` script in your local environment. -The advanced options not shown in the section above **cannot be specified in -the GitHub Action**, hence the need for a local run (refer to the "Prerequisites -(for local environment)" section). -For the explanation of the available parameters, run: -``` -python3.9 generation/new_client_hermetic_build/add-new-client-config.py generate --help -``` - -After you run the script, you will see that the `generation_config.yaml` file -was modified (or the script exited because the library already existed). - -Please create a PR explaining what commands you used and make sure the -add-new-client-config.py arguments were listed). - -### Special case example: Google Maps - -Sometimes, a library generation requires special handling for -Maven coordinates or API ID, especially when the library is not -specific to Google Cloud. The table below is the summary of the -special cases: - -| API paths | `--api_shortname` | `--distribution-name` | -|-------------------|-----------------------------|--------------------------------------------------------| -| google/shopping/* | `shopping-` | `com.google.shipping:google-shopping-` | -| google/maps/* | `maps-` | `com.google.maps:google-maps-` | - -where `` is the value from Cloud Drop file. diff --git a/generation/new_client_hermetic_build/add-new-client-config.py b/generation/new_client_hermetic_build/add-new-client-config.py deleted file mode 100644 index 8213c59d3bf2..000000000000 --- a/generation/new_client_hermetic_build/add-new-client-config.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Script for enabling generation of a new library -Appends a new library to `generation_config.yaml` - -Some of the logic is copied from new-client.py as -a transition step while we test the hermetic build scripts -""" - -import click -import os -import re -import sys -import shutil -from typing import List -from ruamel.yaml import YAML -from git import Repo - -yaml = YAML() - -script_dir = os.path.dirname(os.path.realpath(__file__)) - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--api-shortname", - required=True, - type=str, - prompt="Service name? (e.g. automl)", - help="Name for the new directory name and (default) artifact name", -) -@click.option( - "--name-pretty", - required=True, - type=str, - prompt="Pretty name? (e.g. 'Cloud AutoML')", - help="The human-friendly name that appears in README.md", -) -@click.option( - "--proto-path", - required=True, - type=str, - default=None, - help="Path to proto file from the root of the googleapis repository to the " - "directory that contains the proto files (including the version). " - "For example, to generate the library for 'google/maps/routing/v2', " - "then you specify this value as 'google/maps/routing/v2'", -) -@click.option( - "--product-docs", - required=True, - type=str, - prompt="Product Documentation URL", - help="Documentation URL that appears in README.md", -) -@click.option( - "--rest-docs", - type=str, - help="If it exists, link to the REST Documentation for a service", -) -@click.option( - "--rpc-docs", - type=str, - help="If it exists, link to the RPC Documentation for a service", -) -@click.option( - "--api-description", - required=True, - type=str, - prompt="Description for README. The first sentence is prefixed by the " - "pretty name", - help="Description that appears in README.md", -) -@click.option( - "--library-name", - type=str, - default=None, - help="The directory name of the new library. By default it's " - "java-", -) -@click.option( - "--distribution-name", - type=str, - help="Maven coordinates of the generated library. By default it's " - "com.google.cloud:google-cloud-. " - "It cannot be set at the same time with group_id", -) -@click.option( - "--release-level", - type=click.Choice(["stable", "preview"]), - default="preview", - show_default=True, - help="A label that appears in repo-metadata.json. The first library " - "generation is always 'preview'.", -) -@click.option( - "--api-id", - type=str, - help="The value of the apiid parameter used in README.md It has link to " - "https://console.cloud.google.com/flows/enableapi?apiid=", -) -@click.option( - "--requires-billing", - type=bool, - default=True, - show_default=True, - help="Based on this value, README.md explains whether billing setup is " - "needed or not.", -) -@click.option( - "--group-id", - type=str, - help="The group ID of the artifact when distribution_name is not set. " - "It cannot be set at the same time as distribution_name", -) -@click.option( - "--library-type", - type=str, - default="GAPIC_AUTO", - show_default=True, - help="A label that appears in repo-metadata.json to tell how the library is " - "maintained or generated", -) -@click.option("--api-reference", type=str, help="API reference for this library") -@click.option("--codeowner-team", type=str, help="Team owning this library") -@click.option( - "--excluded-dependencies", - type=str, - help="Comma-separated list of dependencies excluded from this library. The modules specified " - "here will not be added to the poms when postprocessing.", -) -@click.option( - "--excluded-poms", - type=str, - help="Comma-separated list of pom files excluded from postprocessing.", -) -@click.option( - "--googleapis-committish", - type=str, - help="Committish of googleapis/googleapis to get the protos from. It will " - "override the repo-level committish and is not subject to automatic updates", -) -@click.option("--issue-tracker", type=str, help="Issue tracker of the library") -@click.option( - "--extra-versioned-modules", - type=str, - help="Extra modules of the libraries that will be managed via versions.txt", -) -def add_new_library( - api_shortname, - name_pretty, - proto_path, - product_docs, - rest_docs, - rpc_docs, - api_description, - library_name, - distribution_name, - release_level, - api_id, - requires_billing, - group_id, - library_type, - api_reference, - codeowner_team, - excluded_dependencies, - excluded_poms, - googleapis_committish, - issue_tracker, - extra_versioned_modules, -): - output_name = library_name.split("java-")[-1] if library_name else api_shortname - if distribution_name is None: - group_id = "com.google.cloud" - distribution_name = f"{group_id}:google-cloud-{output_name}" - elif group_id: - sys.exit("--group-id and --distribution-name are mutually exclusive options") - else: - group_id = distribution_name.split(":")[0] - - distribution_name_short = re.split(r"[:\/]", distribution_name)[-1] - cloud_api = distribution_name_short.startswith("google-cloud-") - - if api_id is None: - api_id = f"{api_shortname}.googleapis.com" - - if not product_docs.startswith("https"): - sys.exit( - f"product_docs must starts with 'https://' - actual value is {product_docs}" - ) - - client_documentation = f"https://cloud.google.com/java/docs/reference/{distribution_name_short}/latest/overview" - - if api_shortname == "": - sys.exit("api_shortname is empty") - - path_to_yaml = os.path.join(script_dir, "..", "..", "generation_config.yaml") - with open(path_to_yaml, "r") as file_stream: - config = yaml.load(file_stream) - - version_re = re.compile(r"v\d[\w\d]*") - is_library_version = lambda p: version_re.match(p.split("/")[-1]) is not None - if not is_library_version(proto_path): - print( - f"WARNING: proto_path '{proto_path}' does not end with a version (e.g., v1). " - "Please ensure this is intentional and that this represents a non-versioned " - "or custom proto path." - ) - - new_library = { - "api_shortname": api_shortname, - "name_pretty": name_pretty, - "product_documentation": product_docs, - "api_description": api_description, - "client_documentation": client_documentation, - "release_level": release_level, - "distribution_name": distribution_name, - "api_id": api_id, - "library_type": library_type, - "group_id": group_id, - "cloud_api": cloud_api, - "GAPICs": [{"proto_path": proto_path}], - } - - __add_item_if_set(new_library, "library_name", library_name) - __add_item_if_set(new_library, "requires_billing", requires_billing) - __add_item_if_set(new_library, "rest_documentation", rest_docs) - __add_item_if_set(new_library, "rpc_documentation", rpc_docs) - __add_item_if_set(new_library, "distribution_name", distribution_name) - __add_item_if_set(new_library, "api_reference", api_reference) - __add_item_if_set(new_library, "codeowner_team", codeowner_team) - __add_item_if_set(new_library, "excluded_dependencies", excluded_dependencies) - __add_item_if_set(new_library, "excluded_poms", excluded_poms) - __add_item_if_set(new_library, "googleapis_commitish", googleapis_committish) - __add_item_if_set(new_library, "issue_tracker", issue_tracker) - __add_item_if_set(new_library, "extra_versioned_modules", extra_versioned_modules) - - config["libraries"].append(new_library) - config["libraries"] = sorted(config["libraries"], key=__compute_library_name) - - with open(path_to_yaml, "w") as file_stream: - yaml.dump(config, file_stream) - - -def __add_item_if_set(target: dict, key: str, value: any) -> None: - if value is not None: - target[key] = value - - -def __compute_library_name(library: dict) -> str: - if "library_name" in library: - return f'java-{library["library_name"]}' - return f'java-{library["api_shortname"]}' - - - -if __name__ == "__main__": - main() diff --git a/generation/new_client_hermetic_build/generate-arguments.py b/generation/new_client_hermetic_build/generate-arguments.py deleted file mode 100644 index 94af2aa8b825..000000000000 --- a/generation/new_client_hermetic_build/generate-arguments.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Helper script to generate arguments for new-client.py from environment variables -""" -import sys -import os - -required_args = [ - 'API_SHORTNAME', - 'NAME_PRETTY', - 'API_DESCRIPTION', - 'PROTO_PATH', - 'PRODUCT_DOCS', -] - -optional_args = [ - 'REST_DOCS', - 'RPC_DOCS', - 'LIBRARY_NAME', - 'DISTRIBUTION_NAME', -] - -def main() -> None: - result = '' - queries = [(required_args, True), (optional_args, False)] - for (args, is_required) in queries: - for arg in args: - value = os.getenv(arg) - result += __to_python_arg(arg, value, is_required) - - print(result) - -def __to_python_arg(arg: str, value: str, is_required: bool) -> str: - if value is None or value == '': - if is_required: - sys.exit(f'required env var {arg} is not set') - return "" - - return f"--{arg.lower().replace('_','-')} \"{value}\" " - - - -if __name__ == "__main__": - main() diff --git a/generation/new_client_hermetic_build/requirements.in b/generation/new_client_hermetic_build/requirements.in deleted file mode 100644 index 344219b0320d..000000000000 --- a/generation/new_client_hermetic_build/requirements.in +++ /dev/null @@ -1,3 +0,0 @@ -click -ruamel.yaml -GitPython diff --git a/generation/new_client_hermetic_build/requirements.txt b/generation/new_client_hermetic_build/requirements.txt deleted file mode 100644 index 14c7951f8d86..000000000000 --- a/generation/new_client_hermetic_build/requirements.txt +++ /dev/null @@ -1,66 +0,0 @@ -click==8.1.7 \ - --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ - --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de -ruamel.yaml==0.18.6 \ - --hash=sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636 \ - --hash=sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b -ruamel.yaml.clib==0.2.8 \ - --hash=sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d \ - --hash=sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001 \ - --hash=sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462 \ - --hash=sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9 \ - --hash=sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe \ - --hash=sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b \ - --hash=sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b \ - --hash=sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615 \ - --hash=sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62 \ - --hash=sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15 \ - --hash=sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b \ - --hash=sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1 \ - --hash=sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9 \ - --hash=sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675 \ - --hash=sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899 \ - --hash=sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7 \ - --hash=sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7 \ - --hash=sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312 \ - --hash=sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa \ - --hash=sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91 \ - --hash=sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b \ - --hash=sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6 \ - --hash=sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3 \ - --hash=sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334 \ - --hash=sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5 \ - --hash=sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3 \ - --hash=sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe \ - --hash=sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c \ - --hash=sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed \ - --hash=sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337 \ - --hash=sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880 \ - --hash=sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f \ - --hash=sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d \ - --hash=sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248 \ - --hash=sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d \ - --hash=sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf \ - --hash=sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512 \ - --hash=sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069 \ - --hash=sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb \ - --hash=sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942 \ - --hash=sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d \ - --hash=sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31 \ - --hash=sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92 \ - --hash=sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5 \ - --hash=sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28 \ - --hash=sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d \ - --hash=sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1 \ - --hash=sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2 \ - --hash=sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875 \ - --hash=sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412 -GitPython==3.1.42 \ - --hash=sha256:1bf9cd7c9e7255f77778ea54359e54ac22a72a5b51288c457c881057b7bb9ecd \ - --hash=sha256:2d99869e0fef71a73cbd242528105af1d6c1b108c60dfabd994bf292f76c3ceb -gitdb==4.0.1 \ - --hash=sha256:24572287933a9e5bf45260cd7a5629e5b3b190ec3c2c6a5d5e4125dd06ff4027 \ - --hash=sha256:d82c6b76ce8496c5209adf1c0ab969a6e1a8a31510ceb5f57a206fc7c77c0fea -smmap==3.0.1 \ - --hash=sha256:171484fe62793e3626c8b05dd752eb2ca01854b0c55a1efc0dc4210fccb65446 \ - --hash=sha256:5fead614cf2de17ee0707a8c6a5f2aa5a2fc6c698c70993ba42f515485ffda78