diff --git a/.github/workflows/bqmonitor-pr.yml b/.github/workflows/bqmonitor-pr.yml new file mode 100644 index 0000000000..6ed8fe5879 --- /dev/null +++ b/.github/workflows/bqmonitor-pr.yml @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: BigQuery Anomaly Detection PR + +on: + pull_request: + branches: + - 'main' + paths: + - 'python/src/main/python/bigquery-anomaly-detection/**' + - 'python/src/test/python/bigquery-anomaly-detection/**' + - 'python/src/main/java/**/BigQueryAnomalyDetection*.java' + - 'python/src/test/java/**/BigQueryAnomalyDetection*.java' + - '.github/workflows/bqmonitor-pr.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + IT_REGION: us-west2 + +permissions: + actions: write + checks: write + contents: read + pull-requests: read + statuses: write + +jobs: + python_unit_tests: + name: Python Unit Tests + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies and run tests + working-directory: python + run: | + pip install -r src/test/python/bigquery-anomaly-detection/requirements-test.txt + pip install -e src/main/python/bigquery-anomaly-detection + python -m unittest discover \ + -s src/test/python/bigquery-anomaly-detection \ + -p '*_test.py' \ + -v + java_build: + name: Build + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: | + ./cicd/run-build \ + --modules-to-build="YAML" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_tests: + name: Integration Tests + needs: [python_unit_tests, java_build] + timeout-minutes: 120 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --modules-to-build="YAML" \ + --it-region="${{ env.IT_REGION }}" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-connect-2" \ + --test="BigQueryAnomalyDetectionIT" + - name: Upload Integration Tests Report + uses: actions/upload-artifact@v6 + if: always() + with: + name: surefire-integration-test-results + path: | + **/surefire-reports/TEST-*.xml + **/surefire-reports/*.html + **/surefire-reports/html/** + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + if: always() diff --git a/@ b/@ new file mode 100644 index 0000000000..e649932ba3 --- /dev/null +++ b/@ @@ -0,0 +1,47 @@ +initial + +# Please enter the commit message for your changes. Lines starting +# with '#' will be ignored, and an empty message aborts the commit. +# +# Date: Mon Mar 9 09:12:57 2026 -0400 +# +# On branch bqmonitor +# Your branch is up to date with 'origin/bqmonitor'. +# +# Changes to be committed: +# modified: plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java +# modified: plugins/core-plugin/src/main/resources/Dockerfile-template-python +# modified: plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java +# modified: plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java +# new file: python/README_BigQuery_Anomaly_Detection.md +# new file: python/default_base_bqmonitor_requirements.txt +# modified: python/generate_all_dependencies.sh +# new file: python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java +# new file: python/src/main/python/bigquery-anomaly-detection/main.py +# new file: python/src/main/python/bigquery-anomaly-detection/pyproject.toml +# new file: python/src/main/python/bigquery-anomaly-detection/requirements.txt +# new file: python/src/main/python/bigquery-anomaly-detection/setup.py +# new file: python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/__init__.py +# new file: python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py +# new file: python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py +# new file: python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py +# new file: python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py +# modified: python/src/main/python/job-builder-server/requirements.txt +# modified: python/src/main/python/streaming-llm/requirements.txt +# modified: python/src/main/python/word-count-python/requirements.txt +# modified: python/src/main/python/yaml-template/requirements.txt +# new file: python/src/test/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetectionIT.java +# modified: v2/googlecloud-and-mongodb/src/main/resources/requirements.txt +# modified: v2/googlecloud-to-elasticsearch/src/main/resources/requirements.txt +# modified: v2/googlecloud-to-googlecloud/src/main/resources/requirements.txt +# modified: v2/googlecloud-to-splunk/src/main/resources/requirements.txt +# modified: v2/pubsub-binary-to-bigquery/src/main/resources/requirements.txt +# modified: yaml/src/main/python/requirements.txt +# +# Untracked files: +# diff.txt +# launcher_logs.txt +# v1/logs.txt +# v1/src/main/java/com/google/cloud/teleport/templates/unwrap-hec-payload.js +# yaml/src/test/python/logs.txt +# diff --git a/diff.txt b/diff.txt new file mode 100644 index 0000000000..15730a28cf --- /dev/null +++ b/diff.txt @@ -0,0 +1,8318 @@ +diff --git a/.github/workflows/bqmonitor-pr.yml b/.github/workflows/bqmonitor-pr.yml +new file mode 100644 +index 000000000..6ed8fe587 +--- /dev/null ++++ b/.github/workflows/bqmonitor-pr.yml +@@ -0,0 +1,113 @@ ++# Copyright 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# https://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++name: BigQuery Anomaly Detection PR ++ ++on: ++ pull_request: ++ branches: ++ - 'main' ++ paths: ++ - 'python/src/main/python/bigquery-anomaly-detection/**' ++ - 'python/src/test/python/bigquery-anomaly-detection/**' ++ - 'python/src/main/java/**/BigQueryAnomalyDetection*.java' ++ - 'python/src/test/java/**/BigQueryAnomalyDetection*.java' ++ - '.github/workflows/bqmonitor-pr.yml' ++ workflow_dispatch: ++ ++concurrency: ++ group: ${{ github.workflow }}-${{ github.ref }} ++ cancel-in-progress: true ++ ++env: ++ MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error ++ IT_REGION: us-west2 ++ ++permissions: ++ actions: write ++ checks: write ++ contents: read ++ pull-requests: read ++ statuses: write ++ ++jobs: ++ python_unit_tests: ++ name: Python Unit Tests ++ timeout-minutes: 15 ++ runs-on: ubuntu-latest ++ steps: ++ - name: Checkout Code ++ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 ++ - name: Set up Python ++ uses: actions/setup-python@v5 ++ with: ++ python-version: '3.11' ++ - name: Install dependencies and run tests ++ working-directory: python ++ run: | ++ pip install -r src/test/python/bigquery-anomaly-detection/requirements-test.txt ++ pip install -e src/main/python/bigquery-anomaly-detection ++ python -m unittest discover \ ++ -s src/test/python/bigquery-anomaly-detection \ ++ -p '*_test.py' \ ++ -v ++ java_build: ++ name: Build ++ timeout-minutes: 60 ++ runs-on: [self-hosted, it] ++ steps: ++ - name: Checkout Code ++ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 ++ - name: Setup Environment ++ id: setup-env ++ uses: ./.github/actions/setup-env ++ - name: Run Build ++ run: | ++ ./cicd/run-build \ ++ --modules-to-build="YAML" ++ - name: Cleanup Java Environment ++ uses: ./.github/actions/cleanup-java-env ++ java_integration_tests: ++ name: Integration Tests ++ needs: [python_unit_tests, java_build] ++ timeout-minutes: 120 ++ runs-on: [self-hosted, it] ++ steps: ++ - name: Checkout Code ++ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 ++ - name: Setup Environment ++ id: setup-env ++ uses: ./.github/actions/setup-env ++ - name: Run Integration Tests ++ run: | ++ ./cicd/run-it-tests \ ++ --modules-to-build="YAML" \ ++ --it-region="${{ env.IT_REGION }}" \ ++ --it-project="cloud-teleport-testing" \ ++ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ ++ --it-private-connectivity="datastream-connect-2" \ ++ --test="BigQueryAnomalyDetectionIT" ++ - name: Upload Integration Tests Report ++ uses: actions/upload-artifact@v6 ++ if: always() ++ with: ++ name: surefire-integration-test-results ++ path: | ++ **/surefire-reports/TEST-*.xml ++ **/surefire-reports/*.html ++ **/surefire-reports/html/** ++ retention-days: 1 ++ - name: Cleanup Java Environment ++ uses: ./.github/actions/cleanup-java-env ++ if: always() +diff --git a/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java b/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java +index 9d96e4ecf..0d3f4bd2a 100644 +--- a/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java ++++ b/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java +@@ -210,6 +210,8 @@ public class DockerfileGenerator { + + this.parameters.put("filesToCopy", ""); + this.parameters.put("directoriesToCopy", ""); ++ this.parameters.put("setupFileEnv", ""); ++ this.parameters.put("setupInstall", ""); + this.parameters.put("commandSpec", ""); + } + +@@ -337,6 +339,25 @@ public class DockerfileGenerator { + return addStringParameter("directoriesToCopy", directories.toString()); + } + ++ /** ++ * Configures the Dockerfile to install a Python package via {@code pip install .} at build time ++ * and sets {@code FLEX_TEMPLATE_PYTHON_SETUP_FILE} so the Beam stager packages the source code ++ * for distribution to workers. The absolute path avoids issues with {@code os.chdir()}. ++ * ++ * @param setupFile the setup file name (e.g. "setup.py"). ++ * @return this {@link Builder}. ++ */ ++ public Builder setSetupFile(String setupFile) { ++ Preconditions.checkArgument(!Strings.isNullOrEmpty(setupFile)); ++ String workDir = ++ (String) this.parameters.getOrDefault("workingDirectory", DEFAULT_WORKING_DIRECTORY); ++ addParameter( ++ "setupFileEnv", ++ "ENV FLEX_TEMPLATE_PYTHON_SETUP_FILE=\"" + workDir + "/" + setupFile + "\""); ++ addParameter("setupInstall", "RUN pip install --no-cache-dir ."); ++ return this; ++ } ++ + /** + * For XLANG templates, set the {@code DATAFLOW_JAVA_COMMAND_SPEC} env variable to the command + * spec location on the image. +diff --git a/plugins/core-plugin/src/main/resources/Dockerfile-template-python b/plugins/core-plugin/src/main/resources/Dockerfile-template-python +index 8f09eb1e8..995bfc8a0 100644 +--- a/plugins/core-plugin/src/main/resources/Dockerfile-template-python ++++ b/plugins/core-plugin/src/main/resources/Dockerfile-template-python +@@ -3,17 +3,27 @@ FROM ${basePythonContainerImage} + ARG WORKDIR=${workingDirectory} + RUN mkdir -p $WORKDIR + ${filesToCopy} ++${directoriesToCopy} + WORKDIR $WORKDIR + +-ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE=requirements.txt ++# Do NOT use ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE here. All deps are preinstalled ++# at build time. That env var triggers the Beam stager to re-resolve requirements on the ++# launcher VM, where platform tag mismatches cause pip to compile C extensions (e.g. numpy) ++# from source, freezing or timing out the launcher. ++# FLEX_TEMPLATE_PYTHON_SETUP_FILE (via setupFileEnv) IS needed to stage custom code to workers. ++# TODO: For templates without setup.py that need non-beam deps on workers, split requirements ++# into a build-only lockfile and a stager-safe extra-requirements file. ++ARG REQUIREMENTS_FILE=requirements.txt + ENV FLEX_TEMPLATE_PYTHON_PY_FILE=main.py ++${setupFileEnv} + + RUN if ! [ -f requirements.txt ] ; then >&2 echo "error: no requirements.txt file found" && exit 1 ; fi + + # Set up custom PyPi repository, if applicable + ${airlockConfig} + +-RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE +-RUN pip download --require-hashes --no-cache-dir --dest /tmp/dataflow-requirements-cache -r $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE ++RUN pip install -U --require-hashes -r $REQUIREMENTS_FILE ++${setupInstall} ++RUN pip download --require-hashes --no-cache-dir --dest /tmp/dataflow-requirements-cache -r $REQUIREMENTS_FILE + +-ENTRYPOINT ${entryPoint} +\ No newline at end of file ++ENTRYPOINT ${entryPoint} +diff --git a/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java b/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java +index b8e51a818..8823b8f38 100644 +--- a/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java ++++ b/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java +@@ -58,8 +58,7 @@ public class DockerfileGeneratorTest { + assertTrue(outputFile.exists()); + String fileContents = Files.asCharSource(outputFile, StandardCharsets.UTF_8).read(); + assertThat(fileContents).contains("FROM " + BASE_PYTHON_CONTAINER_IMAGE); +- assertThat(fileContents) +- .contains("RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE"); ++ assertThat(fileContents).contains("RUN pip install -U --require-hashes -r $REQUIREMENTS_FILE"); + assertThat(fileContents) + .contains(String.format("ENTRYPOINT [\"%s\"]", PYTHON_LAUNCHER_ENTRYPOINT)); + } +@@ -80,8 +79,7 @@ public class DockerfileGeneratorTest { + assertTrue(outputFile.exists()); + String fileContents = Files.asCharSource(outputFile, StandardCharsets.UTF_8).read(); + assertThat(fileContents).contains("FROM a python container image"); +- assertThat(fileContents) +- .contains("RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE"); ++ assertThat(fileContents).contains("RUN pip install -U --require-hashes -r $REQUIREMENTS_FILE"); + assertThat(fileContents).contains("COPY main.py requirements.txt $WORKDIR/"); + assertThat(fileContents).contains("ENTRYPOINT [\"python/entry/point\"]"); + } +diff --git a/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java b/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java +index 8b4e21c3a..208a20dde 100644 +--- a/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java ++++ b/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java +@@ -1008,7 +1008,7 @@ public class TemplatesStageMojo extends TemplatesBaseMojo { + } + + List entryPoint = List.of(definition.getTemplateAnnotation().entryPoint()); +- if (entryPoint.isEmpty()) { ++ if (entryPoint.isEmpty() || (entryPoint.size() == 1 && entryPoint.get(0).isEmpty())) { + entryPoint = List.of(pythonTemplateLauncherEntryPoint); + } + +@@ -1053,25 +1053,31 @@ public class TemplatesStageMojo extends TemplatesBaseMojo { + String dockerfilePath = dockerfileContainer + "/Dockerfile"; + File dockerfile = new File(dockerfilePath); + if (!dockerfile.exists()) { +- List filesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy()); ++ List allFilesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy()); ++ if (allFilesToCopy.isEmpty()) { ++ allFilesToCopy = List.of("main.py", "requirements.txt"); ++ } ++ ++ // Separate flat files from directories ++ List filesToCopy = new ArrayList<>(); ++ Set directoriesToCopy = new HashSet<>(); ++ for (String f : allFilesToCopy) { ++ File source = new File(dockerfileContainer + "/" + f); ++ if (source.isDirectory()) { ++ directoriesToCopy.add(f); ++ } else { ++ filesToCopy.add(f); ++ } ++ } + if (filesToCopy.isEmpty()) { + filesToCopy = List.of("main.py", "requirements.txt"); + } ++ + List entryPoint = List.of(definition.getTemplateAnnotation().entryPoint()); +- if (entryPoint.isEmpty()) { ++ if (entryPoint.isEmpty() || (entryPoint.size() == 1 && entryPoint.get(0).isEmpty())) { + entryPoint = List.of(pythonTemplateLauncherEntryPoint); + } + +- // Copy in requirements.txt if present +- File sourceRequirements = new File(outputClassesDirectory.getPath() + "/requirements.txt"); +- File destRequirements = new File(dockerfileContainer + "/requirements.txt"); +- if (sourceRequirements.exists()) { +- Files.copy( +- sourceRequirements.toPath(), +- destRequirements.toPath(), +- StandardCopyOption.REPLACE_EXISTING); +- } +- + // Generate Dockerfile + LOG.info("Generating dockerfile " + dockerfilePath); + DockerfileGenerator.Builder dockerfileBuilder = +@@ -1079,11 +1085,21 @@ public class TemplatesStageMojo extends TemplatesBaseMojo { + definition.getTemplateAnnotation().type(), + beamVersion, + containerName, +- targetDirectory) ++ outputClassesDirectory) + .setBasePythonContainerImage(basePythonContainerImage) + .setFilesToCopy(filesToCopy) + .setEntryPoint(entryPoint); + ++ if (!directoriesToCopy.isEmpty()) { ++ dockerfileBuilder.setDirectoriesToCopy(directoriesToCopy); ++ } ++ ++ // Configure setup.py support if present ++ File setupFile = new File(dockerfileContainer + "/setup.py"); ++ if (setupFile.exists()) { ++ dockerfileBuilder.setSetupFile("setup.py"); ++ } ++ + // Set Airlock parameters + if (internalMaven) { + dockerfileBuilder +diff --git a/pom.xml b/pom.xml +index a35bfc36d..278089092 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -615,6 +615,7 @@ + + + **/KafkaToKafkaIT.java ++ **/BigQueryAnomalyDetectionIT.java + + ${direct-runner.tests} + +@@ -878,6 +879,7 @@ + + + **/KafkaToKafkaIT.java ++ **/BigQueryAnomalyDetectionIT.java + + + ${direct-runner.tests}, +diff --git a/python/README_BigQuery_Anomaly_Detection.md b/python/README_BigQuery_Anomaly_Detection.md +new file mode 100644 +index 000000000..9e0fdb727 +--- /dev/null ++++ b/python/README_BigQuery_Anomaly_Detection.md +@@ -0,0 +1,328 @@ ++ ++BigQuery Anomaly Detection (Experimental) ++--- ++> **Note:** This template is experimental and may change without notice. ++ ++A streaming Dataflow Flex Template that monitors a BigQuery table for anomalies ++in real time. The pipeline reads CDC (Change Data Capture) data from BigQuery, ++computes a configurable windowed metric, runs statistical anomaly detection, and ++publishes detected anomalies to a Pub/Sub topic. ++ ++Supported anomaly detectors: **ZScore**, **IQR**, **RobustZScore** (from ++Apache Beam's `apache_beam.ml.anomaly` module), and **Threshold** (a simple ++fixed-threshold alerter based on a boolean expression). ++ ++## Parameters ++ ++### Required parameters ++ ++* **table**: BigQuery table to monitor. Format: `project:dataset.table`. ++* **metric_spec**: JSON string defining the metric computation (see [Metric Spec Reference](#metric-spec-reference) below). ++* **detector_spec**: JSON string defining the anomaly detector (see [Detector Spec Reference](#detector-spec-reference) below). ++* **topic**: Pub/Sub topic for anomaly results. Full path: `projects//topics/`. ++ ++### Optional parameters ++* **poll_interval_sec**: Seconds between BigQuery CDC polls. Default: `60`. ++* **change_function**: BigQuery change function: `APPENDS` or `CHANGES`. Default: `APPENDS`. ++* **buffer_sec**: Safety buffer behind `now()` in seconds. Default: `15`. ++* **start_offset_sec**: Start reading from this many seconds ago. Default: `60`. ++* **duration_sec**: How long to run in seconds. `0` means run forever. Default: `0`. ++* **temp_dataset**: BigQuery dataset for temp tables. If unset, auto-created. ++* **log_all_results**: Log all anomaly detection results (normal, outlier, warmup) at WARNING level. Default: `false`. ++* **sink_table**: BigQuery table to write all anomaly detection results to. Format: `project:dataset.table`. If unset, results are not written to BigQuery. ++* **decompress_shards**: Number of shards for CDC Arrow batch decompression fan-out. Spreads decompression CPU across workers. `0` disables fan-out (decode inline). Default: `400`. ++* **fanout_strategy**: Parallelism strategy for global (non-keyed) metric aggregation: `sharded`, `hotkey_fanout`, or `none`. Ignored when `group_by` is set. Default: `sharded`. See [Fanout Strategies](#fanout-strategies). ++* **fanout**: Number of shards for `sharded` or `hotkey_fanout` strategies. Ignored for `none`. Default: `400`. ++ ++## Metric Spec Reference ++ ++The `metric_spec` parameter is a JSON string that defines how raw rows are ++aggregated into a single numeric value for anomaly detection. ++ ++```json ++{ ++ "aggregation": { ++ "window": { ++ "type": "fixed", ++ "size_seconds": 3600 ++ }, ++ "group_by": ["field1", "field2"], ++ "measures": [ ++ {"field": "amount", "agg": "SUM", "alias": "total"} ++ ] ++ }, ++ "derived_fields": [ ++ {"name": "is_success", "expression": "1 if status == 'success' else 0"} ++ ], ++ "measure_combiner": {"expression": "clicks / impressions"} ++} ++``` ++ ++| Field | Required | Description | ++|---|---|---| ++| `aggregation` | Yes | Windowed aggregation configuration. | ++| `aggregation.window.type` | Yes | `fixed` or `sliding`. | ++| `aggregation.window.size_seconds` | Yes | Window size in seconds. | ++| `aggregation.window.period_seconds` | Sliding only | Slide period in seconds. | ++| `aggregation.group_by` | No | Field names for grouping. Omit for global aggregation. | ++| `aggregation.measures` | Yes | List of aggregation measures. | ++| `aggregation.measures[].field` | Yes | Input field name (ignored for `COUNT`). | ++| `aggregation.measures[].agg` | Yes | `SUM`, `COUNT`, `MIN`, `MAX`, or `MEAN`. | ++| `aggregation.measures[].alias` | Yes | Output name for this measure. | ++| `derived_fields` | No | Pre-aggregation computed columns. | ++| `measure_combiner` | When >1 measure | Post-aggregation expression combining measure aliases. | ++ ++Expressions support: `+`, `-`, `*`, `/`, `//`, `%`, `**`, comparisons, ++`and/or/not`, `if/else`, safe builtins (`abs`, `min`, `max`, `round`), ++and parentheses. Bare names are field references. ++ ++## Detector Spec Reference ++ ++```json ++{"type": "ZScore"} ++{"type": "ZScore", "config": {"window_size": 500}} ++{"type": "ZScore", "config": {"threshold_criterion": {"type": "FixedThreshold", "config": {"cutoff": 10}}}} ++``` ++ ++| Detector | Description | Default threshold | ++|---|---|---| ++| `ZScore` | `\|value - mean\| / stdev` | 3 | ++| `IQR` | Interquartile Range | 1.5 | ++| `RobustZScore` | Modified Z-Score using median/MAD | 3.5 | ++| `Threshold` | Fixed threshold alert via boolean expression | N/A | ++ ++**Threshold** evaluates a boolean expression against the metric `value` and ++fires an alert (label=1) when the expression is true: ++ ++```json ++{"type": "Threshold", "expression": "value >= 100"} ++{"type": "Threshold", "expression": "value > 100 or value < -100"} ++{"type": "Threshold", "expression": "value <= 0.01"} ++``` ++ ++The `window_size` shorthand (default: 1000) sets the history buffer for all ++internal statistical trackers. ++ ++### Threshold overrides ++ ++```json ++{"type": "FixedThreshold", "config": {"cutoff": 10}} ++{"type": "QuantileThreshold", "config": {"quantile": 0.95}} ++``` ++ ++## Pub/Sub Output ++ ++Detected anomalies (label == 1) are published to the configured Pub/Sub topic ++as JSON messages: ++ ++```json ++{ ++ "event_description": "Anomaly detected value=1234.56 score=4.2 in window=2026-03-19T12:00:00.000000Z-2026-03-19T13:00:00.000000Z", ++ "agent_id": "ZScore", ++ "key": "(campaign_a, chrome)" ++} ++``` ++ ++The `key` field is only present for grouped (keyed) metrics. ++ ++Set `--log_all_results` to log all results (normal, outlier, warmup) at ++WARNING level in the Dataflow worker logs. ++ ++## Fanout Strategies ++ ++For global aggregation (no `group_by`), all elements are combined under a ++single key. At high throughput this creates a bottleneck on the single reducer's ++streaming state I/O. The `fanout_strategy` pipeline parameter controls how ++elements are distributed across intermediate reducers before the final merge. ++ ++| Strategy | How it works | Best for | ++|---|---|---| ++| `sharded` (default) | Per-element random sharding into N shard keys. Stage 1 `CombinePerKey` reduces each shard independently. Stage 2 `CombineGlobally` merges N partial accumulators. | High-throughput global aggregation (e.g., 1M+ rows/sec). Uniform distribution regardless of bundle count. | ++| `hotkey_fanout` | Beam's built-in `CombineGlobally.with_fanout(N)`. Per-bundle nonce sharding — all elements in one bundle go to the same shard. Better mapper-side pre-combine (PGBK) table efficiency. | When upstream provides many small bundles, or for moderate throughput where PGBK efficiency matters. | ++| `none` | Plain `CombineGlobally` with no fanout. Relies on Dataflow's combiner lifting (PGBK) for mapper-side pre-combining. | Low throughput, or when upstream already provides enough parallel bundles (e.g., `decompress_shards`) and streaming state I/O is not a bottleneck. | ++ ++## Getting Started ++ ++### Requirements ++ ++* Java 17 ++* Maven 3.9.9+ ++* Python 3.11+ ++* [gcloud CLI](https://cloud.google.com/sdk/gcloud), and execution of the ++ following commands: ++ * `gcloud auth login` ++ * `gcloud auth application-default login` ++ ++### Required IAM Permissions ++ ++The **worker service account** (used by Dataflow workers) needs: ++ ++| Role | Reason | ++|---|---| ++| `roles/storage.objectAdmin` | Read/write GCS for staging artifacts and temp files (see note below) | ++| `roles/dataflow.developer` | Create and manage Dataflow jobs | ++| `roles/bigquery.dataOwner` | Create/delete temp datasets and tables, read CDC data | ++| `roles/bigquery.jobUser` | Run BigQuery query jobs | ++| `roles/pubsub.editor` | Publish anomaly alerts and verify topic exists | ++ ++The **user or CI account** that launches the template also needs ++`roles/iam.serviceAccountUser` on the worker service account to impersonate it. ++ ++> **Note:** If you pre-create the temp dataset with `--temp_dataset`, you can ++> scope `roles/bigquery.dataOwner` to just the source and temp datasets ++> instead of project-wide, and use `roles/bigquery.dataEditor` if dataset ++> deletion is not needed. ++ ++> **Note:** Dataflow auto-creates a default staging bucket ++> (`dataflow-staging-{region}-{project_number}`) on first use in a region. ++> If this bucket does not exist, the service account needs ++> `roles/storage.admin` (or the bucket must be pre-created). Once the ++> bucket exists, `roles/storage.objectAdmin` is sufficient. ++ ++### Building the Plugins ++ ++The Maven plugins must be installed before staging: ++ ++```shell ++mvn install -pl plugins/core-plugin,plugins/templates-maven-plugin -am -DskipTests -q ++``` ++ ++### Staging the Template ++ ++```shell ++export PROJECT= ++export BUCKET_NAME= ++ ++mvn clean package -PtemplatesStage \ ++ -DskipTests \ ++ -DprojectId="$PROJECT" \ ++ -DbucketName="$BUCKET_NAME" \ ++ -DstagePrefix="templates" \ ++ -DtemplateName="BigQuery_Anomaly_Detection" \ ++ -pl python ++``` ++ ++This builds the Docker image, pushes it to `gcr.io/$PROJECT/bigquery-anomaly-detection`, ++and writes the template spec to `gs://$BUCKET_NAME/templates/flex/BigQuery_Anomaly_Detection`. ++ ++### Running the Template ++ ++```shell ++export PROJECT= ++export BUCKET_NAME= ++export REGION=us-central1 ++ ++gcloud dataflow flex-template run "bq-anomaly-$(date +%Y%m%d-%H%M%S)" \ ++ --project "$PROJECT" \ ++ --region "$REGION" \ ++ --template-file-gcs-location "gs://$BUCKET_NAME/templates/flex/BigQuery_Anomaly_Detection" \ ++ --parameters table="$PROJECT:my_dataset.my_table" \ ++ --parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":60},"measures":[{"field":"amount","agg":"SUM","alias":"revenue"}]}}' \ ++ --parameters detector_spec='{"type":"ZScore"}' \ ++ --parameters topic="projects/$PROJECT/topics/bqmonitor-anomalies" \ ++ --parameters duration_sec="300" ++``` ++ ++Or run directly from the container image (skipping the GCS spec file): ++ ++```shell ++gcloud dataflow flex-template run "bq-anomaly-test" \ ++ --image "gcr.io/$PROJECT/bigquery-anomaly-detection:templates" \ ++ --project "$PROJECT" \ ++ --region "$REGION" \ ++ --sdk-language PYTHON \ ++ --parameters table="$PROJECT:my_dataset.my_table" \ ++ --parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":60},"measures":[{"field":"amount","agg":"SUM","alias":"revenue"}]}}' \ ++ --parameters detector_spec='{"type":"ZScore"}' ++``` ++ ++### Regenerating Pinned Dependencies ++ ++The `requirements.txt` contains pinned and hashed dependencies. To regenerate ++after changing base dependencies: ++ ++```shell ++# Edit python/default_base_bqmonitor_requirements.txt, then: ++sh python/generate_all_dependencies.sh ++``` ++ ++### Running Integration Tests ++ ++The integration tests stage the template, launch it against real BigQuery and ++Pub/Sub resources, and verify end-to-end anomaly detection. ++ ++```shell ++export PROJECT= ++export REGION=us-east5 ++export BUCKET_NAME= ++ ++# Build plugins first (one-time). ++mvn install -pl plugins/core-plugin,plugins/templates-maven-plugin -am -DskipTests -q ++ ++# Run the integration tests. ++mvn verify -PtemplatesIntegrationTests \ ++ -Dproject="$PROJECT" \ ++ -Dregion="$REGION" \ ++ -DartifactBucket="gs://$BUCKET_NAME" \ ++ -pl python \ ++ -Dtest=BigQueryAnomalyDetectionIT ++``` ++ ++To run a single test method: ++ ++```shell ++mvn verify -PtemplatesIntegrationTests \ ++ -Dproject="$PROJECT" \ ++ -Dregion="$REGION" \ ++ -DartifactBucket="gs://$BUCKET_NAME" \ ++ -pl python \ ++ -Dtest=BigQueryAnomalyDetectionIT#testDetectsAnomalyAndPublishesToPubSub ++``` ++ ++The test service account needs all roles listed in ++[Required IAM Permissions](#required-iam-permissions) plus the ability to ++create and delete test resources (Pub/Sub topics/subscriptions, BigQuery ++datasets/tables). ++ ++## Examples ++ ++### Simple SUM metric ++ ++```shell ++--parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' ++--parameters detector_spec='{"type":"ZScore"}' ++``` ++ ++### Grouped ratio metric (CTR) ++ ++```shell ++--parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":60},"group_by":["campaign_type","browser"],"measures":[{"field":"is_click","agg":"SUM","alias":"clicks"},{"field":"is_click","agg":"COUNT","alias":"impressions"}]},"measure_combiner":{"expression":"clicks / impressions"}}' ++--parameters detector_spec='{"type":"ZScore"}' ++``` ++ ++### Derived field + custom threshold ++ ++```shell ++--parameters metric_spec='{"derived_fields":[{"name":"is_success","expression":"1 if status == '"'"'success'"'"' else 0"}],"aggregation":{"window":{"type":"fixed","size_seconds":60},"group_by":["brand"],"measures":[{"field":"is_success","agg":"SUM","alias":"successes"},{"field":"is_success","agg":"COUNT","alias":"total"}]},"measure_combiner":{"expression":"successes / total"}}' ++--parameters detector_spec='{"type":"ZScore","config":{"threshold_criterion":{"type":"FixedThreshold","config":{"cutoff":10}}}}' ++``` ++ ++## Project Structure ++ ++``` ++python/src/main/python/bigquery-anomaly-detection/ ++ main.py # Entry point ++ setup.py # Package configuration ++ pyproject.toml # Build system config ++ requirements.txt # Pinned dependencies (generated) ++ src/bqmonitor/ ++ __init__.py ++ pipeline.py # Pipeline construction and options ++ cdc.py # BigQuery CDC reader (ReadBigQueryChangeHistory) ++ metric.py # MetricSpec and ComputeMetric PTransform ++ safe_eval.py # Safe expression evaluation (Expr) ++ ++python/src/main/java/.../BigQueryAnomalyDetection.java # Template metadata ++python/src/test/java/.../BigQueryAnomalyDetectionIT.java # Integration test ++python/default_base_bqmonitor_requirements.txt # Base dependencies ++``` +diff --git a/python/default_base_bqmonitor_requirements.txt b/python/default_base_bqmonitor_requirements.txt +new file mode 100644 +index 000000000..abd9ea899 +--- /dev/null ++++ b/python/default_base_bqmonitor_requirements.txt +@@ -0,0 +1,3 @@ ++apache-beam[gcp]==2.71.0 ++google-cloud-bigquery-storage ++setuptools +diff --git a/python/generate_all_dependencies.sh b/python/generate_all_dependencies.sh +index 1e0035cdd..d02635972 100755 +--- a/python/generate_all_dependencies.sh ++++ b/python/generate_all_dependencies.sh +@@ -20,6 +20,7 @@ set -e + SCRIPTPATH=$(dirname "$0") + + sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/streaming-llm/base_requirements.txt $SCRIPTPATH/../python/src/main/python/streaming-llm/requirements.txt ++sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_bqmonitor_requirements.txt $SCRIPTPATH/../python/src/main/python/bigquery-anomaly-detection/requirements_all.txt + # Generate a base set of dependencies to use for any templates without special dependencies + mkdir -p $SCRIPTPATH/__build__/ + sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_python_requirements.txt $SCRIPTPATH/__build__/default_python_requirements.txt +diff --git a/python/pom.xml b/python/pom.xml +index af8a7bf57..56bff10c5 100644 +--- a/python/pom.xml ++++ b/python/pom.xml +@@ -83,6 +83,74 @@ + + + ++ ++ bqmonitorPythonTests ++ ++ false ++ ++ ++ ++ ++ org.codehaus.mojo ++ exec-maven-plugin ++ ${exec-maven-plugin.version} ++ ++ ++ bqmonitor-pip-install ++ test-compile ++ ++ exec ++ ++ ++ pip ++ ++ install ++ -r ++ src/test/python/bigquery-anomaly-detection/requirements-test.txt ++ ++ ++ ++ ++ bqmonitor-pip-install-pkg ++ test-compile ++ ++ exec ++ ++ ++ pip ++ ++ install ++ -e ++ src/main/python/bigquery-anomaly-detection ++ ++ ++ ++ ++ bqmonitor-python-test ++ test ++ ++ exec ++ ++ ++ python ++ ${project.basedir} ++ ++ -m ++ unittest ++ discover ++ -s ++ src/test/python/bigquery-anomaly-detection ++ -p ++ *_test.py ++ -v ++ ++ ++ ++ ++ ++ ++ ++ + + templatesValidate + +diff --git a/python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java b/python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java +new file mode 100644 +index 000000000..88a3bd82c +--- /dev/null ++++ b/python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java +@@ -0,0 +1,181 @@ ++/* ++ * Copyright (C) 2026 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); you may not ++ * use this file except in compliance with the License. You may obtain a copy of ++ * the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++ * License for the specific language governing permissions and limitations under ++ * the License. ++ */ ++package com.google.cloud.teleport.templates.python; ++ ++import com.google.cloud.teleport.metadata.Template; ++import com.google.cloud.teleport.metadata.Template.TemplateType; ++import com.google.cloud.teleport.metadata.TemplateCategory; ++import com.google.cloud.teleport.metadata.TemplateParameter; ++ ++/** Template class for BigQuery Anomaly Detection in Python. */ ++@Template( ++ name = "BigQuery_Anomaly_Detection", ++ category = TemplateCategory.STREAMING, ++ type = TemplateType.PYTHON, ++ displayName = "BigQuery Anomaly Detection", ++ description = ++ "[Experimental] Real-time anomaly detection on BigQuery change data (CDC). " ++ + "Reads streaming APPENDS/CHANGES data from a BigQuery table, " ++ + "computes a configurable windowed metric, runs anomaly detection " ++ + "(ZScore, IQR, or RobustZScore), and publishes anomalies to Pub/Sub.", ++ preview = true, ++ flexContainerName = "bigquery-anomaly-detection", ++ filesToCopy = {"main.py", "setup.py", "pyproject.toml", "requirements_all.txt", "src"}, ++ contactInformation = "https://cloud.google.com/support", ++ streaming = true) ++public interface BigQueryAnomalyDetection { ++ ++ @TemplateParameter.Text( ++ order = 1, ++ name = "table", ++ description = "BigQuery Table", ++ helpText = "BigQuery table to monitor. Format: project:dataset.table", ++ regexes = {"^[a-zA-Z0-9_-]+:[a-zA-Z0-9_]+\\.[a-zA-Z0-9_]+$"}) ++ String getTable(); ++ ++ @TemplateParameter.Text( ++ order = 2, ++ name = "metric_spec", ++ description = "Metric Specification (JSON)", ++ helpText = ++ "JSON string defining the metric computation. " ++ + "Example: {\"aggregation\":{\"window\":{\"type\":\"fixed\"," ++ + "\"size_seconds\":3600},\"measures\":[{\"field\":\"amount\",\"agg\":\"SUM\"," ++ + "\"alias\":\"total\"}]}}") ++ String getMetricSpec(); ++ ++ @TemplateParameter.Text( ++ order = 3, ++ name = "detector_spec", ++ description = "Detector Specification (JSON)", ++ helpText = ++ "JSON string defining the anomaly detector. " ++ + "Example: {\"type\":\"ZScore\"} or " ++ + "{\"type\":\"ZScore\",\"config\":{\"threshold_criterion\":{\"type\":\"FixedThreshold\"," ++ + "\"config\":{\"cutoff\":10}}}}") ++ String getDetectorSpec(); ++ ++ @TemplateParameter.Integer( ++ order = 5, ++ optional = true, ++ name = "poll_interval_sec", ++ description = "Poll Interval (seconds)", ++ helpText = "Seconds between BigQuery CDC polls. Default: 60.") ++ Integer getPollIntervalSec(); ++ ++ @TemplateParameter.Text( ++ order = 6, ++ optional = true, ++ name = "change_function", ++ description = "Change Function", ++ helpText = "BigQuery change function: APPENDS or CHANGES. Default: APPENDS.", ++ regexes = {"^(APPENDS|CHANGES)$"}) ++ String getChangeFunction(); ++ ++ @TemplateParameter.Integer( ++ order = 7, ++ optional = true, ++ name = "buffer_sec", ++ description = "Buffer (seconds)", ++ helpText = "Safety buffer behind now() in seconds. Default: 15.") ++ Integer getBufferSec(); ++ ++ @TemplateParameter.Integer( ++ order = 8, ++ optional = true, ++ name = "start_offset_sec", ++ description = "Start Offset (seconds)", ++ helpText = "Start reading from this many seconds ago. Default: 60.") ++ Integer getStartOffsetSec(); ++ ++ @TemplateParameter.Integer( ++ order = 9, ++ optional = true, ++ name = "duration_sec", ++ description = "Duration (seconds)", ++ helpText = "How long to run in seconds. 0 means run forever. Default: 0.") ++ Integer getDurationSec(); ++ ++ @TemplateParameter.Text( ++ order = 10, ++ optional = true, ++ name = "temp_dataset", ++ description = "Temp Dataset", ++ helpText = "BigQuery dataset for temp tables. If unset, auto-created.") ++ String getTempDataset(); ++ ++ @TemplateParameter.Text( ++ order = 4, ++ name = "topic", ++ description = "Pub/Sub Topic", ++ helpText = ++ "Pub/Sub topic for anomaly results. " + "Full path: projects//topics/.") ++ String getTopic(); ++ ++ @TemplateParameter.Boolean( ++ order = 11, ++ optional = true, ++ name = "log_all_results", ++ description = "Log All Results", ++ helpText = ++ "Log all anomaly detection results (normal, outlier, warmup) " ++ + "at WARNING level. Default: false.") ++ Boolean getLogAllResults(); ++ ++ @TemplateParameter.Text( ++ order = 12, ++ optional = true, ++ name = "sink_table", ++ description = "Sink BigQuery Table", ++ helpText = ++ "BigQuery table to write all anomaly detection results to. " ++ + "Format: project:dataset.table. If unset, results are not written to BigQuery.", ++ regexes = {"^[a-zA-Z0-9_-]+:[a-zA-Z0-9_]+\\.[a-zA-Z0-9_]+$"}) ++ String getSinkTable(); ++ ++ @TemplateParameter.Integer( ++ order = 13, ++ optional = true, ++ name = "decompress_shards", ++ description = "Decompress Shards", ++ helpText = ++ "Number of shards for CDC Arrow batch decompression fan-out. " ++ + "Spreads decompression CPU across workers. " ++ + "0 disables fan-out (decode inline). Default: 400.") ++ Integer getDecompressShards(); ++ ++ @TemplateParameter.Text( ++ order = 14, ++ optional = true, ++ name = "fanout_strategy", ++ description = "Fanout Strategy", ++ helpText = ++ "Parallelism strategy for global (non-keyed) metric aggregation: " ++ + "sharded, hotkey_fanout, or none. " ++ + "Ignored when group_by is set. Default: sharded.", ++ regexes = {"^(sharded|hotkey_fanout|none)$"}) ++ String getFanoutStrategy(); ++ ++ @TemplateParameter.Integer( ++ order = 15, ++ optional = true, ++ name = "fanout", ++ description = "Fanout Shards", ++ helpText = ++ "Number of shards for sharded or hotkey_fanout strategies. " ++ + "Ignored for none. Default: 400.") ++ Integer getFanout(); ++} +diff --git a/python/src/main/python/bigquery-anomaly-detection/main.py b/python/src/main/python/bigquery-anomaly-detection/main.py +new file mode 100644 +index 000000000..afd7bf028 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/main.py +@@ -0,0 +1,25 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Flex Template entry point for bqmonitor.""" ++ ++import logging ++ ++from bqmonitor.pipeline import run ++ ++if __name__ == '__main__': ++ logging.getLogger().setLevel(logging.INFO) ++ run() +diff --git a/python/src/main/python/bigquery-anomaly-detection/pyproject.toml b/python/src/main/python/bigquery-anomaly-detection/pyproject.toml +new file mode 100644 +index 000000000..c36dd5d36 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/pyproject.toml +@@ -0,0 +1,13 @@ ++[build-system] ++requires = ["setuptools>=64", "wheel"] ++build-backend = "setuptools.build_meta" ++ ++[project] ++name = "bqmonitor" ++version = "0.1.0" ++description = "BigQuery anomaly monitoring pipeline (Dataflow Flex Template)" ++requires-python = ">=3.11" ++dependencies = [ ++ "apache-beam[gcp]==2.71.0", ++ "google-cloud-bigquery-storage", ++] +diff --git a/python/src/main/python/bigquery-anomaly-detection/requirements_all.txt b/python/src/main/python/bigquery-anomaly-detection/requirements_all.txt +new file mode 100644 +index 000000000..ecae6a536 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/requirements_all.txt +@@ -0,0 +1,2579 @@ ++# Copyright 2025 Google Inc. All Rights Reserved. ++ ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++ ++# http://www.apache.org/licenses/LICENSE-2.0 ++ ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++# Autogenerated requirements file for Apache Beam container image. ++# From the templates base directory to update, ++# run: sh python/generate_all_dependencies.sh ++# Do not edit manually, adjust the base requirements file, and regenerate the list. ++ ++# See [maintainers-guide](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/contributor-docs/maintainers-guide.md#validating-and-upgrading-beam-versions) for more information. ++ ++# ++# This file is autogenerated by pip-compile with Python 3.11 ++# by the following command: ++# ++# pip-compile --allow-unsafe --generate-hashes --output-file=python/../python/src/main/python/bigquery-anomaly-detection/requirements.txt python/default_base_bqmonitor_requirements.txt ++# ++aiofiles==25.1.0 \ ++ --hash=sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2 \ ++ --hash=sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695 ++ # via cloud-sql-python-connector ++aiohappyeyeballs==2.6.1 \ ++ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ ++ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 ++ # via aiohttp ++aiohttp==3.13.3 \ ++ --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf \ ++ --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c \ ++ --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c \ ++ --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 \ ++ --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f \ ++ --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 \ ++ --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 \ ++ --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf \ ++ --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 \ ++ --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 \ ++ --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 \ ++ --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 \ ++ --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d \ ++ --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea \ ++ --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 \ ++ --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 \ ++ --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 \ ++ --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 \ ++ --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 \ ++ --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 \ ++ --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a \ ++ --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e \ ++ --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 \ ++ --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 \ ++ --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 \ ++ --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd \ ++ --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a \ ++ --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 \ ++ --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 \ ++ --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 \ ++ --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f \ ++ --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 \ ++ --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 \ ++ --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 \ ++ --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 \ ++ --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce \ ++ --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 \ ++ --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 \ ++ --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 \ ++ --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 \ ++ --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b \ ++ --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 \ ++ --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c \ ++ --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 \ ++ --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 \ ++ --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f \ ++ --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b \ ++ --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e \ ++ --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 \ ++ --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 \ ++ --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 \ ++ --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d \ ++ --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 \ ++ --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 \ ++ --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce \ ++ --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 \ ++ --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 \ ++ --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c \ ++ --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf \ ++ --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 \ ++ --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 \ ++ --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e \ ++ --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 \ ++ --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a \ ++ --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 \ ++ --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a \ ++ --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 \ ++ --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 \ ++ --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba \ ++ --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df \ ++ --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 \ ++ --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 \ ++ --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 \ ++ --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 \ ++ --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc \ ++ --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 \ ++ --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 \ ++ --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 \ ++ --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 \ ++ --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 \ ++ --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 \ ++ --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 \ ++ --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 \ ++ --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 \ ++ --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 \ ++ --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc \ ++ --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 \ ++ --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 \ ++ --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 \ ++ --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 \ ++ --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 \ ++ --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 \ ++ --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c \ ++ --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 \ ++ --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 \ ++ --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a \ ++ --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 \ ++ --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 \ ++ --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 \ ++ --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d \ ++ --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 \ ++ --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d \ ++ --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f \ ++ --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f \ ++ --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 \ ++ --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 \ ++ --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 \ ++ --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 \ ++ --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af \ ++ --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 \ ++ --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 \ ++ --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 \ ++ --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c \ ++ --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 \ ++ --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 \ ++ --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 \ ++ --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 \ ++ --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 \ ++ --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa \ ++ --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 ++ # via cloud-sql-python-connector ++aiosignal==1.4.0 \ ++ --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ ++ --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 ++ # via aiohttp ++annotated-types==0.7.0 \ ++ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ ++ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 ++ # via pydantic ++anyio==4.12.1 \ ++ --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ ++ --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c ++ # via ++ # google-genai ++ # httpx ++apache-beam[gcp]==2.71.0 \ ++ --hash=sha256:044841032ef190a7ad69a9d4ca4b23c104a310d08c47a1f5faefcf830c9e5520 \ ++ --hash=sha256:192b00d13de8eb06241c5332ecef7a9a947758e2103b07d6726848ba9f0b5a49 \ ++ --hash=sha256:1cdaf7e502da67f674ecf8dd8cec21252bd1b2678a5d18b873a45635cf0e7cec \ ++ --hash=sha256:28c6eeb05b688dcc503fce84075fcd03a73bbd9e449e70521f2efb47a932bcea \ ++ --hash=sha256:317f5495c3266b9146263dbb881110b56b015fbc7e2f1e27eb9932b2bf28a94c \ ++ --hash=sha256:3705d824d462aee4bf162318eb0ef1ca767064e73aa4f1ba14d741cc12c19143 \ ++ --hash=sha256:43ed7ae3dbecf67af2ad412b86d160fc6177d19fc6e59ed18aee4a84355858db \ ++ --hash=sha256:515064493c478e92a87618f46c8b8c2143ce244317db683dc3d824fda37b0db5 \ ++ --hash=sha256:5ca7fca47ae39b5e6497c39bca303d11c200fdfae6b352e5e481a59a9b886f75 \ ++ --hash=sha256:78c2f8e88014555984a7a21bcb63479e135b958428d178d45699a4154ae84634 \ ++ --hash=sha256:78e3e913275bd1c1aac1ecc90af78fb65915908671b6e39d60a3a31de3438782 \ ++ --hash=sha256:81766907e53a5feddb2d1b5553c6f1154ff7cae67e548b4c2726e299334572bf \ ++ --hash=sha256:8189d2e1d314a7dc8f3456bae4c7641637d302490e1af93db3aa6ba45d716b70 \ ++ --hash=sha256:83be2fce3726529f221c8d99f844f64d68494b2bad438852f96f02f2c0e8cac8 \ ++ --hash=sha256:8e1cbc386cf8c0d740b3b2847cb7c99481672ed036b57c11eb2f41d049800b40 \ ++ --hash=sha256:a11147b82260d69b19021b32a65da044d38f65195ec2a66460ccad80649106b5 \ ++ --hash=sha256:a14fb6972de7113dfbe6bba967de1a3a5c60228a96b96eb32a675762f83d659b \ ++ --hash=sha256:a358a7e689e1acb903ec5f545ed22b674fb6cbb17424518630412cba3a627937 \ ++ --hash=sha256:a7967a1d75daec31e9d03705304ad4e7e5bcad266dd5e8bad98a68e76ebb368f \ ++ --hash=sha256:af5a9acf850b8430440f8e6f687650c252dd7d0b929fbef2d84ce79087f6bb6b \ ++ --hash=sha256:b3acb72a5afdc15abe696e37915cbce91d7a0672fda2658c2185d8ea4684d4e3 \ ++ --hash=sha256:c015aa7ee75cabc58277b19317429fc3ed08752173d6750b2212260190505c7f \ ++ --hash=sha256:d4a3b4008ca3966f426a8580535e2227387518a2d62c3928c4e3d5a6ca23dd8a \ ++ --hash=sha256:de890d820ae365eddcbe522e61816a967ab9d5be501fb56435e0d8a8c571408e \ ++ --hash=sha256:e06fb7fd4f5aa9d16bb8d8d30d9c24fc255cfc9be510188bfab0b11f398cc515 ++ # via -r python/default_base_bqmonitor_requirements.txt ++asn1crypto==1.5.1 \ ++ --hash=sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c \ ++ --hash=sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67 ++ # via scramp ++attrs==25.4.0 \ ++ --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ ++ --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 ++ # via aiohttp ++backports-tarfile==1.2.0 \ ++ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ ++ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 ++ # via jaraco-context ++beartype==0.22.9 \ ++ --hash=sha256:8f82b54aa723a2848a56008d18875f91c1db02c32ef6a62319a002e3e25a975f \ ++ --hash=sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2 ++ # via apache-beam ++betterproto==2.0.0b7 \ ++ --hash=sha256:1b1458ca5278d519bcd62556a4c236f998a91d503f0f71c67b0b954747052af2 \ ++ --hash=sha256:401ab8055e2f814e77b9c88a74d0e1ae3d1e8a969cced6aeb1b59f71ad63fbd2 ++ # via envoy-data-plane ++cachetools==6.2.6 \ ++ --hash=sha256:16c33e1f276b9a9c0b49ab5782d901e3ad3de0dd6da9bf9bcd29ac5672f2f9e6 \ ++ --hash=sha256:8c9717235b3c651603fff0076db52d6acbfd1b338b8ed50256092f7ce9c85bda ++ # via apache-beam ++certifi==2026.2.25 \ ++ --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ ++ --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 ++ # via ++ # httpcore ++ # httpx ++ # requests ++cffi==2.0.0 \ ++ --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ ++ --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ ++ --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ ++ --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ ++ --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ ++ --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ ++ --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ ++ --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ ++ --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ ++ --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ ++ --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ ++ --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ ++ --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ ++ --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ ++ --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ ++ --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ ++ --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ ++ --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ ++ --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ ++ --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ ++ --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ ++ --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ ++ --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ ++ --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ ++ --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ ++ --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ ++ --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ ++ --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ ++ --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ ++ --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ ++ --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ ++ --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ ++ --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ ++ --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ ++ --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ ++ --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ ++ --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ ++ --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ ++ --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ ++ --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ ++ --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ ++ --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ ++ --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ ++ --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ ++ --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ ++ --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ ++ --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ ++ --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ ++ --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ ++ --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ ++ --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ ++ --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ ++ --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ ++ --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ ++ --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ ++ --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ ++ --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ ++ --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ ++ --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ ++ --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ ++ --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ ++ --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ ++ --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ ++ --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ ++ --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ ++ --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ ++ --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ ++ --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ ++ --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ ++ --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ ++ --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ ++ --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ ++ --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ ++ --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ ++ --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ ++ --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ ++ --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ ++ --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ ++ --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ ++ --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ ++ --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ ++ --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ ++ --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ ++ --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf ++ # via cryptography ++charset-normalizer==3.4.5 \ ++ --hash=sha256:014837af6fabf57121b6254fa8ade10dceabc3528b27b721a64bbc7b8b1d4eb4 \ ++ --hash=sha256:01a1ed54b953303ca7e310fafe0fe347aab348bd81834a0bcd602eb538f89d66 \ ++ --hash=sha256:0294916d6ccf2d069727d65973c3a1ca477d68708db25fd758dd28b0827cff54 \ ++ --hash=sha256:02a9d1b01c1e12c27883b0c9349e0bcd9ae92e727ff1a277207e1a262b1cbf05 \ ++ --hash=sha256:036c079aa08a6a592b82487f97c60b439428320ed1b2ea0b3912e99d30c77765 \ ++ --hash=sha256:039215608ac7b358c4da0191d10fc76868567fbf276d54c14721bdedeb6de064 \ ++ --hash=sha256:0625665e4ebdddb553ab185de5db7054393af8879fb0c87bd5690d14379d6819 \ ++ --hash=sha256:0a45e504f5e1be0bd385935a8e1507c442349ca36f511a47057a71c9d1d6ea9e \ ++ --hash=sha256:0b362bcd27819f9c07cbf23db4e0e8cd4b44c5ecd900c2ff907b2b92274a7412 \ ++ --hash=sha256:0c300cefd9b0970381a46394902cd18eaf2aa00163f999590ace991989dcd0fc \ ++ --hash=sha256:1088345bcc93c58d8d8f3d783eca4a6e7a7752bbff26c3eee7e73c597c191c2e \ ++ --hash=sha256:10b473fc8dca1c3ad8559985794815f06ca3fc71942c969129070f2c3cdf7281 \ ++ --hash=sha256:131716d6786ad5e3dc542f5cc6f397ba3339dc0fb87f87ac30e550e8987756af \ ++ --hash=sha256:14498a429321de554b140013142abe7608f9d8ccc04d7baf2ad60498374aefa2 \ ++ --hash=sha256:149ec69866c3d6c2fb6f758dbc014ecb09f30b35a5ca90b6a8a2d4e54e18fdfe \ ++ --hash=sha256:165c7b21d19365464e8f70e5ce5e12524c58b48c78c1f5a57524603c1ab003f8 \ ++ --hash=sha256:1827734a5b308b65ac54e86a618de66f935a4f63a8a462ff1e19a6788d6c2262 \ ++ --hash=sha256:19092dde50335accf365cce21998a1c6dd8eafd42c7b226eb54b2747cdce2fac \ ++ --hash=sha256:1a374cc0b88aa710e8865dc1bd6edb3743c59f27830f0293ab101e4cf3ce9f85 \ ++ --hash=sha256:1d1401945cb77787dbd3af2446ff2d75912327c4c3a1526ab7955ecf8600687c \ ++ --hash=sha256:1f2da5cbb9becfcd607757a169e38fb82aa5fd86fae6653dea716e7b613fe2cf \ ++ --hash=sha256:259cd1ca995ad525f638e131dbcc2353a586564c038fc548a3fe450a91882139 \ ++ --hash=sha256:2820a98460c83663dd8ec015d9ddfd1e4879f12e06bb7d0500f044fb477d2770 \ ++ --hash=sha256:28269983f25a4da0425743d0d257a2d6921ea7d9b83599d4039486ec5b9f911d \ ++ --hash=sha256:2b970382e4a36bed897c19f310f31d7d13489c11b4f468ddfba42d41cddfb918 \ ++ --hash=sha256:2da4eedcb6338e2321e831a0165759c0c620e37f8cd044a263ff67493be8ffb3 \ ++ --hash=sha256:30987f4a8ed169983f93e1be8ffeea5214a779e27ed0b059835c7afe96550ad7 \ ++ --hash=sha256:30a2b1a48478c3428d047ed9690d57c23038dac838a87ad624c85c0a78ebeb39 \ ++ --hash=sha256:340810d34ef83af92148e96e3e44cb2d3f910d2bf95e5618a5c467d9f102231d \ ++ --hash=sha256:3f64c6bf8f32f9133b668c7f7a7cbdbc453412bc95ecdbd157f3b1e377a92990 \ ++ --hash=sha256:4167a621a9a1a986c73777dbc15d4b5eac8ac5c10393374109a343d4013ec765 \ ++ --hash=sha256:4354e401eb6dab9aed3c7b4030514328a6c748d05e1c3e19175008ca7de84fb1 \ ++ --hash=sha256:4481e6da1830c8a1cc0b746b47f603b653dadb690bcd851d039ffaefe70533aa \ ++ --hash=sha256:4b8551b6e6531e156db71193771c93bda78ffc4d1e6372517fe58ad3b91e4659 \ ++ --hash=sha256:4cd966c2559f501c6fd69294d082c2934c8dd4719deb32c22961a5ac6db0df1d \ ++ --hash=sha256:50bcbca6603c06a1dcc7b056ed45c37715fb5d2768feb3bcd37d2313c587a5b9 \ ++ --hash=sha256:530beedcec9b6e027e7a4b6ce26eed36678aa39e17da85e6e03d7bd9e8e9d7c9 \ ++ --hash=sha256:568e3c34b58422075a1b49575a6abc616d9751b4d61b23f712e12ebb78fe47b2 \ ++ --hash=sha256:573ef5814c4b7c0d59a7710aa920eaaaef383bd71626aa420fba27b5cab92e8d \ ++ --hash=sha256:58ad8270cfa5d4bef1bc85bd387217e14ff154d6630e976c6f56f9a040757475 \ ++ --hash=sha256:597d10dec876923e5c59e48dbd366e852eacb2b806029491d307daea6b917d7c \ ++ --hash=sha256:5bcb3227c3d9aaf73eaaab1db7ccd80a8995c509ee9941e2aae060ca6e4e5d81 \ ++ --hash=sha256:5cffde4032a197bd3b42fd0b9509ec60fb70918d6970e4cc773f20fc9180ca67 \ ++ --hash=sha256:5fea359734b140d0d6741189fea5478c6091b54ffc69d7ce119e0a05637d8c99 \ ++ --hash=sha256:60d68e820af339df4ae8358c7a2e7596badeb61e544438e489035f9fbf3246a5 \ ++ --hash=sha256:610f72c0ee565dfb8ae1241b666119582fdbfe7c0975c175be719f940e110694 \ ++ --hash=sha256:65a126fb4b070d05340a84fc709dd9e7c75d9b063b610ece8a60197a291d0adf \ ++ --hash=sha256:65b3c403a5b6b8034b655e7385de4f72b7b244869a22b32d4030b99a60593eca \ ++ --hash=sha256:66dee73039277eb35380d1b82cccc69cc82b13a66f9f4a18da32d573acf02b7c \ ++ --hash=sha256:708c7acde173eedd4bfa4028484426ba689d2103b28588c513b9db2cd5ecde9c \ ++ --hash=sha256:728c6a963dfab66ef865f49286e45239384249672cd598576765acc2a640a636 \ ++ --hash=sha256:754f96058e61a5e22e91483f823e07df16416ce76afa4ebf306f8e1d1296d43f \ ++ --hash=sha256:75dfd1afe0b1647449e852f4fb428195a7ed0588947218f7ba929f6538487f02 \ ++ --hash=sha256:75ee9c1cce2911581a70a3c0919d8bccf5b1cbc9b0e5171400ec736b4b569497 \ ++ --hash=sha256:76a9d0de4d0eab387822e7b35d8f89367dd237c72e82ab42b9f7bf5e15ada00f \ ++ --hash=sha256:77be992288f720306ab4108fe5c74797de327f3248368dfc7e1a916d6ed9e5a2 \ ++ --hash=sha256:7ad83b8f9379176c841f8865884f3514d905bcd2a9a3b210eaa446e7d2223e4d \ ++ --hash=sha256:8197abe5ca1ffb7d91e78360f915eef5addff270f8a71c1fc5be24a56f3e4873 \ ++ --hash=sha256:82cc7c2ad42faec8b574351f8bc2a0c049043893853317bd9bb309f5aba6cb5a \ ++ --hash=sha256:8a28afb04baa55abf26df544e3e5c6534245d3daa5178bc4a8eeb48202060d0e \ ++ --hash=sha256:8b78d8a609a4b82c273257ee9d631ded7fac0d875bdcdccc109f3ee8328cfcb1 \ ++ --hash=sha256:8ce11cd4d62d11166f2b441e30ace226c19a3899a7cf0796f668fba49a9fb123 \ ++ --hash=sha256:8fff79bf5978c693c9b1a4d71e4a94fddfb5fe744eb062a318e15f4a2f63a550 \ ++ --hash=sha256:92263f7eca2f4af326cd20de8d16728d2602f7cfea02e790dcde9d83c365d7cc \ ++ --hash=sha256:93b3b2cc5cf1b8743660ce77a4f45f3f6d1172068207c1defc779a36eea6bb36 \ ++ --hash=sha256:95adae7b6c42a6c5b5b559b1a99149f090a57128155daeea91732c8d970d8644 \ ++ --hash=sha256:97ab7787092eb9b50fb47fa04f24c75b768a606af1bcba1957f07f128a7219e4 \ ++ --hash=sha256:9db5e3fcdcee89a78c04dffb3fe33c79f77bd741a624946db2591c81b2fc85b0 \ ++ --hash=sha256:a118e2e0b5ae6b0120d5efa5f866e58f2bb826067a646431da4d6a2bdae7950e \ ++ --hash=sha256:a2aecdb364b8a1802afdc7f9327d55dad5366bc97d8502d0f5854e50712dbc5f \ ++ --hash=sha256:a66aa5022bf81ab4b1bebfb009db4fd68e0c6d4307a1ce5ef6a26e5878dfc9e4 \ ++ --hash=sha256:a68766a3c58fde7f9aaa22b3786276f62ab2f594efb02d0a1421b6282e852e98 \ ++ --hash=sha256:aa2f963b4da26daf46231d9b9e0e2c9408a751f8f0d0f44d2de56d3caf51d294 \ ++ --hash=sha256:aa92ec1102eaff840ccd1021478af176a831f1bccb08e526ce844b7ddda85c22 \ ++ --hash=sha256:ac59c15e3f1465f722607800c68713f9fbc2f672b9eb649fe831da4019ae9b23 \ ++ --hash=sha256:ae8b03427410731469c4033934cf473426faff3e04b69d2dfb64a4281a3719f8 \ ++ --hash=sha256:afca7f78067dd27c2b848f1b234623d26b87529296c6c5652168cc1954f2f3b2 \ ++ --hash=sha256:b2d37d78297b39a9eb9eb92c0f6df98c706467282055419df141389b23f93362 \ ++ --hash=sha256:b3e71afc578b98512bfe7bdb822dd6bc57d4b0093b4b6e5487c1e96ad4ace242 \ ++ --hash=sha256:ba20bdf69bd127f66d0174d6f2a93e69045e0b4036dc1ca78e091bcc765830c4 \ ++ --hash=sha256:c108f8619e504140569ee7de3f97d234f0fbae338a7f9f360455071ef9855a95 \ ++ --hash=sha256:c23eb3263356d94858655b3e63f85ac5d50970c6e8febcdde7830209139cc37d \ ++ --hash=sha256:c5af897b45fa606b12464ccbe0014bbf8c09191e0a66aab6aa9d5cf6e77e0c94 \ ++ --hash=sha256:c7a80a9242963416bd81f99349d5f3fce1843c303bd404f204918b6d75a75fd6 \ ++ --hash=sha256:c7e84e0c0005e3bdc1a9211cd4e62c78ba80bc37b2365ef4410cd2007a9047f2 \ ++ --hash=sha256:cace89841c0599d736d3d74a27bc5821288bb47c5441923277afc6059d7fbcb4 \ ++ --hash=sha256:cd2d0f0ec9aa977a27731a3209ebbcacebebaf41f902bd453a928bfd281cf7f8 \ ++ --hash=sha256:d01de5e768328646e6a3fa9e562706f8f6641708c115c62588aef2b941a4f88e \ ++ --hash=sha256:d1028de43596a315e2720a9849ee79007ab742c06ad8b45a50db8cdb7ed4a82a \ ++ --hash=sha256:d27ce22ec453564770d29d03a9506d449efbb9fa13c00842262b2f6801c48cce \ ++ --hash=sha256:d29dd9c016f2078b43d0c357511e87eee5b05108f3dd603423cb389b89813969 \ ++ --hash=sha256:d31f0d1671e1534e395f9eb84a68e0fb670e1edb1fe819a9d7f564ae3bc4e53f \ ++ --hash=sha256:d4eb8ac7469b2a5d64b5b8c04f84d8bf3ad340f4514b98523805cbf46e3b3923 \ ++ --hash=sha256:d5e52d127045d6ae01a1e821acfad2f3a1866c54d0e837828538fabe8d9d1bd6 \ ++ --hash=sha256:d77f97e515688bd615c1d1f795d540f32542d514242067adcb8ef532504cb9ee \ ++ --hash=sha256:d8ed79b8f6372ca4254955005830fd61c1ccdd8c0fac6603e2c145c61dd95db6 \ ++ --hash=sha256:dc57a0baa3eeedd99fafaef7511b5a6ef4581494e8168ee086031744e2679467 \ ++ --hash=sha256:e09f671a54ce70b79a1fc1dc6da3072b7ef7251fadb894ed92d9aa8218465a5f \ ++ --hash=sha256:e22d1059b951e7ae7c20ef6b06afd10fb95e3c41bf3c4fbc874dba113321c193 \ ++ --hash=sha256:e37bd100d2c5d3ba35db9c7c5ba5a9228cbcffe5c4778dc824b164e5257813d7 \ ++ --hash=sha256:e51ae7d81c825761d941962450f50d041db028b7278e7b08930b4541b3e45cb9 \ ++ --hash=sha256:e545b51da9f9af5c67815ca0eb40676c0f016d0b0381c86f20451e35696c5f95 \ ++ --hash=sha256:e6302ca4ae283deb0af68d2fbf467474b8b6aedcd3dab4db187e07f94c109763 \ ++ --hash=sha256:e71bbb595973622b817c042bd943c3f3667e9c9983ce3d205f973f486fec98a7 \ ++ --hash=sha256:ec56a2266f32bc06ed3c3e2a8f58417ce02f7e0356edc89786e52db13c593c98 \ ++ --hash=sha256:ed1a9a204f317ef879b32f9af507d47e49cd5e7f8e8d5d96358c98373314fc60 \ ++ --hash=sha256:ed97c282ee4f994ef814042423a529df9497e3c666dca19be1d4cd1129dc7ade \ ++ --hash=sha256:ed98364e1c262cf5f9363c3eca8c2df37024f52a8fa1180a3610014f26eac51c \ ++ --hash=sha256:ee57b926940ba00bca7ba7041e665cc956e55ef482f851b9b65acb20d867e7a2 \ ++ --hash=sha256:f1d725b754e967e648046f00c4facc42d414840f5ccc670c5670f59f83693e4f \ ++ --hash=sha256:f8102ae93c0bc863b1d41ea0f4499c20a83229f52ed870850892df555187154a \ ++ --hash=sha256:fc1c64934b8faf7584924143eb9db4770bbdb16659626e1a1a4d9efbcb68d947 \ ++ --hash=sha256:ff95a9283de8a457e6b12989de3f9f5193430f375d64297d323a615ea52cbdb3 ++ # via requests ++cloud-sql-python-connector==1.20.0 \ ++ --hash=sha256:aa7c30631c5f455d14d561d7b0b414a97652a1b582a301f5570ba2cea2aa9105 \ ++ --hash=sha256:fdd96153b950040b0252453115604c142922b72cf3636146165a648ac5f6fc30 ++ # via apache-beam ++cryptography==46.0.5 \ ++ --hash=sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72 \ ++ --hash=sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235 \ ++ --hash=sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9 \ ++ --hash=sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356 \ ++ --hash=sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257 \ ++ --hash=sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad \ ++ --hash=sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4 \ ++ --hash=sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c \ ++ --hash=sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614 \ ++ --hash=sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed \ ++ --hash=sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31 \ ++ --hash=sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229 \ ++ --hash=sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0 \ ++ --hash=sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731 \ ++ --hash=sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b \ ++ --hash=sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4 \ ++ --hash=sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4 \ ++ --hash=sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263 \ ++ --hash=sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595 \ ++ --hash=sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1 \ ++ --hash=sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678 \ ++ --hash=sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48 \ ++ --hash=sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76 \ ++ --hash=sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0 \ ++ --hash=sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18 \ ++ --hash=sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d \ ++ --hash=sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d \ ++ --hash=sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1 \ ++ --hash=sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981 \ ++ --hash=sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7 \ ++ --hash=sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82 \ ++ --hash=sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2 \ ++ --hash=sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4 \ ++ --hash=sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663 \ ++ --hash=sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c \ ++ --hash=sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d \ ++ --hash=sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a \ ++ --hash=sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a \ ++ --hash=sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d \ ++ --hash=sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b \ ++ --hash=sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a \ ++ --hash=sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826 \ ++ --hash=sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee \ ++ --hash=sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9 \ ++ --hash=sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648 \ ++ --hash=sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da \ ++ --hash=sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2 \ ++ --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2 \ ++ --hash=sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87 ++ # via ++ # apache-beam ++ # cloud-sql-python-connector ++ # google-auth ++ # secretstorage ++distro==1.9.0 \ ++ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ ++ --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 ++ # via google-genai ++dnspython==2.8.0 \ ++ --hash=sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af \ ++ --hash=sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f ++ # via ++ # cloud-sql-python-connector ++ # pymongo ++docstring-parser==0.17.0 \ ++ --hash=sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912 \ ++ --hash=sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708 ++ # via google-cloud-aiplatform ++envoy-data-plane==0.2.6 \ ++ --hash=sha256:6341768b9cf5d6268baced4d2e8b3429f98664fbbe8958dae69ee25316ae869a \ ++ --hash=sha256:d1541c8cd00677886a2f93696edf9e3589cd4ac680defc66b3013ffb082f274c ++ # via apache-beam ++fastavro==1.12.1 \ ++ --hash=sha256:00650ca533907361edda22e6ffe8cf87ab2091c5d8aee5c8000b0f2dcdda7ed3 \ ++ --hash=sha256:02281432dcb11c78b3280da996eff61ee0eff39c5de06c6e0fbf19275093e6d4 \ ++ --hash=sha256:0714b285160fcd515eb0455540f40dd6dac93bdeacdb03f24e8eac3d8aa51f8d \ ++ --hash=sha256:089e155c0c76e0d418d7e79144ce000524dd345eab3bc1e9c5ae69d500f71b14 \ ++ --hash=sha256:120aaf82ac19d60a1016afe410935fe94728752d9c2d684e267e5b7f0e70f6d9 \ ++ --hash=sha256:123fb221df3164abd93f2d042c82f538a1d5a43ce41375f12c91ce1355a9141e \ ++ --hash=sha256:1f55eef18c41d4476bd32a82ed5dd86aabc3f614e1b66bdb09ffa291612e1670 \ ++ --hash=sha256:1f81011d54dd47b12437b51dd93a70a9aa17b61307abf26542fc3c13efbc6c51 \ ++ --hash=sha256:2de72d786eb38be6b16d556b27232b1bf1b2797ea09599507938cdb7a9fe3e7c \ ++ --hash=sha256:2f285be49e45bc047ab2f6bed040bb349da85db3f3c87880e4b92595ea093b2b \ ++ --hash=sha256:3100ad643e7fa658469a2a2db229981c1a000ff16b8037c0b58ce3ec4d2107e8 \ ++ --hash=sha256:3616e2f0e1c9265e92954fa099db79c6e7817356d3ff34f4bcc92699ae99697c \ ++ --hash=sha256:3b1921ac35f3d89090a5816b626cf46e67dbecf3f054131f84d56b4e70496f45 \ ++ --hash=sha256:4128978b930aaf930332db4b3acc290783183f3be06a241ae4a482f3ed8ce892 \ ++ --hash=sha256:43ded16b3f4a9f1a42f5970c2aa618acb23ea59c4fcaa06680bdf470b255e5a8 \ ++ --hash=sha256:44cbff7518901c91a82aab476fcab13d102e4999499df219d481b9e15f61af34 \ ++ --hash=sha256:469fecb25cba07f2e1bfa4c8d008477cd6b5b34a59d48715e1b1a73f6160097d \ ++ --hash=sha256:509818cb24b98a804fc80be9c5fed90f660310ae3d59382fc811bfa187122167 \ ++ --hash=sha256:5217f773492bac43dae15ff2931432bce2d7a80be7039685a78d3fab7df910bd \ ++ --hash=sha256:546ffffda6610fca672f0ed41149808e106d8272bb246aa7539fa8bb6f117f17 \ ++ --hash=sha256:5aa777b8ee595b50aa084104cd70670bf25a7bbb9fd8bb5d07524b0785ee1699 \ ++ --hash=sha256:632a4e3ff223f834ddb746baae0cc7cee1068eb12c32e4d982c2fee8a5b483d0 \ ++ --hash=sha256:64961ab15b74b7c168717bbece5660e0f3d457837c3cc9d9145181d011199fa7 \ ++ --hash=sha256:6b632b713bc5d03928a87d811fa4a11d5f25cd43e79c161e291c7d3f7aa740fd \ ++ --hash=sha256:780476c23175d2ae457c52f45b9ffa9d504593499a36cd3c1929662bf5b7b14b \ ++ --hash=sha256:78df838351e4dff9edd10a1c41d1324131ffecbadefb9c297d612ef5363c049a \ ++ --hash=sha256:792356d320f6e757e89f7ac9c22f481e546c886454a6709247f43c0dd7058004 \ ++ --hash=sha256:81563e1f93570e6565487cdb01ba241a36a00e58cff9c5a0614af819d1155d8f \ ++ --hash=sha256:83e6caf4e7a8717d932a3b1ff31595ad169289bbe1128a216be070d3a8391671 \ ++ --hash=sha256:9090f0dee63fe022ee9cc5147483366cc4171c821644c22da020d6b48f576b4f \ ++ --hash=sha256:9445da127751ba65975d8e4bdabf36bfcfdad70fc35b2d988e3950cce0ec0e7c \ ++ --hash=sha256:a275e48df0b1701bb764b18a8a21900b24cf882263cb03d35ecdba636bbc830b \ ++ --hash=sha256:a38607444281619eda3a9c1be9f5397634012d1b237142eee1540e810b30ac8b \ ++ --hash=sha256:a7d840ccd9aacada3ddc80fbcc4ea079b658107fe62e9d289a0de9d54e95d366 \ ++ --hash=sha256:a8bc2dcec5843d499f2489bfe0747999108f78c5b29295d877379f1972a3d41a \ ++ --hash=sha256:ac76d6d95f909c72ee70d314b460b7e711d928845771531d823eb96a10952d26 \ ++ --hash=sha256:b6a3462934b20a74f9ece1daa49c2e4e749bd9a35fa2657b53bf62898fba80f5 \ ++ --hash=sha256:b81fc04e85dfccf7c028e0580c606e33aa8472370b767ef058aae2c674a90746 \ ++ --hash=sha256:b91a0fe5a173679a6c02d53ca22dcaad0a2c726b74507e0c1c2e71a7c3f79ef9 \ ++ --hash=sha256:bec207360f76f0b3de540758a297193c5390e8e081c43c3317f610b1414d8c8f \ ++ --hash=sha256:c0390bfe4a9f8056a75ac6785fbbff8f5e317f5356481d2e29ec980877d2314b \ ++ --hash=sha256:c3d67c47f177e486640404a56f2f50b165fe892cc343ac3a34673b80cc7f1dd6 \ ++ --hash=sha256:cb0337b42fd3c047fcf0e9b7597bd6ad25868de719f29da81eabb6343f08d399 \ ++ --hash=sha256:d71c8aa841ef65cfab709a22bb887955f42934bced3ddb571e98fdbdade4c609 \ ++ --hash=sha256:eaa7ab3769beadcebb60f0539054c7755f63bd9cf7666e2c15e615ab605f89a8 \ ++ --hash=sha256:ed924233272719b5d5a6a0b4d80ef3345fc7e84fc7a382b6232192a9112d38a6 ++ # via apache-beam ++fasteners==0.20 \ ++ --hash=sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8 \ ++ --hash=sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7 ++ # via ++ # apache-beam ++ # google-apitools ++frozenlist==1.8.0 \ ++ --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ ++ --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ ++ --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ ++ --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ ++ --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ ++ --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ ++ --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ ++ --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ ++ --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ ++ --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ ++ --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ ++ --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ ++ --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ ++ --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ ++ --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ ++ --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ ++ --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ ++ --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ ++ --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ ++ --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ ++ --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ ++ --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ ++ --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ ++ --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ ++ --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ ++ --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ ++ --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ ++ --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ ++ --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ ++ --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ ++ --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ ++ --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ ++ --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ ++ --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ ++ --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ ++ --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ ++ --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ ++ --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ ++ --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ ++ --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ ++ --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ ++ --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ ++ --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ ++ --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ ++ --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ ++ --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ ++ --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ ++ --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ ++ --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ ++ --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ ++ --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ ++ --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ ++ --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ ++ --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ ++ --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ ++ --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ ++ --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ ++ --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ ++ --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ ++ --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ ++ --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ ++ --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ ++ --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ ++ --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ ++ --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ ++ --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ ++ --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ ++ --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ ++ --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ ++ --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ ++ --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ ++ --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ ++ --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ ++ --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ ++ --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ ++ --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ ++ --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ ++ --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ ++ --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ ++ --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ ++ --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ ++ --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ ++ --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ ++ --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ ++ --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ ++ --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ ++ --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ ++ --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ ++ --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ ++ --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ ++ --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ ++ --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ ++ --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ ++ --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ ++ --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ ++ --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ ++ --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ ++ --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ ++ --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ ++ --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ ++ --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ ++ --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ ++ --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ ++ --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ ++ --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ ++ --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ ++ --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ ++ --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ ++ --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ ++ --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ ++ --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ ++ --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ ++ --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ ++ --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ ++ --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ ++ --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ ++ --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ ++ --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ ++ --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ ++ --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ ++ --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ ++ --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ ++ --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ ++ --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ ++ --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ ++ --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ ++ --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ ++ --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ ++ --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ ++ --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd ++ # via ++ # aiohttp ++ # aiosignal ++google-api-core[grpc]==2.30.0 \ ++ --hash=sha256:02edfa9fab31e17fc0befb5f161b3bf93c9096d99aed584625f38065c511ad9b \ ++ --hash=sha256:80be49ee937ff9aba0fd79a6eddfde35fe658b9953ab9b79c57dd7061afa8df5 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++ # google-cloud-bigquery ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-core ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++ # google-cloud-storage ++ # google-cloud-videointelligence ++ # google-cloud-vision ++google-apitools==0.5.31 \ ++ --hash=sha256:4af0dd6dd4582810690251f0b57a97c1873dadfda54c5bc195844c8907624170 \ ++ --hash=sha256:6be92c1c3e93485450420bb0e365d47eb4d8a835d03ebe1963dc6da4d39a7b0e ++ # via apache-beam ++google-auth[requests]==2.49.0 \ ++ --hash=sha256:9cc2d9259d3700d7a257681f81052db6737495a1a46b610597f4b8bafe5286ae \ ++ --hash=sha256:f893ef7307f19cf53700b7e2f61b5a6affe3aa0edf9943b13788920ab92d8d87 ++ # via ++ # apache-beam ++ # cloud-sql-python-connector ++ # google-api-core ++ # google-auth-httplib2 ++ # google-cloud-aiplatform ++ # google-cloud-bigquery ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-core ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-storage ++ # google-cloud-videointelligence ++ # google-cloud-vision ++ # google-genai ++ # keyrings-google-artifactregistry-auth ++google-auth-httplib2==0.2.1 \ ++ --hash=sha256:1be94c611db91c01f9703e7f62b0a59bbd5587a95571c7b6fade510d648bc08b \ ++ --hash=sha256:5ef03be3927423c87fb69607b42df23a444e434ddb2555b73b3679793187b7de ++ # via apache-beam ++google-cloud-aiplatform==1.140.0 \ ++ --hash=sha256:e94493a2682b9d17efa7146a53bb3665bf1595c3394fd3d0f45d18f71623fddc \ ++ --hash=sha256:ea7eb1870b4cf600f8c2472102e21c3a1bcaf723d6e49f00ed51bc6b88d54fff ++ # via apache-beam ++google-cloud-bigquery==3.40.1 \ ++ --hash=sha256:75afcfb6e007238fe1deefb2182105249321145ff921784fe7b1de2b4ba24506 \ ++ --hash=sha256:9082a6b8193aba87bed6a2c79cf1152b524c99bb7e7ac33a785e333c09eac868 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++google-cloud-bigquery-storage==2.36.2 \ ++ --hash=sha256:823a73db0c4564e8ad3eedcfd5049f3d5aa41775267863b5627211ec36be2dbf \ ++ --hash=sha256:ad49d8c09ad6cd82da4efe596fcfcdbc1458bf05b93915e3c5c00f1e700ae128 ++ # via ++ # -r python/default_base_bqmonitor_requirements.txt ++ # apache-beam ++google-cloud-bigtable==2.35.0 \ ++ --hash=sha256:f355bfce1f239453ec2bb3839b0f4f9937cf34ef06ef29e1ca63d58fd38d0c50 \ ++ --hash=sha256:f5699012c5fea4bd4bdf7e80e5e3a812a847eb8f41bf8dc2f43095d6d876b83b ++ # via apache-beam ++google-cloud-core==2.5.0 \ ++ --hash=sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc \ ++ --hash=sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963 ++ # via ++ # apache-beam ++ # google-cloud-bigquery ++ # google-cloud-bigtable ++ # google-cloud-datastore ++ # google-cloud-spanner ++ # google-cloud-storage ++google-cloud-datastore==2.23.0 \ ++ --hash=sha256:24a1b1d29b902148fe41b109699f76fd3aa60591e9d547c0f8b87d7bf9ff213f \ ++ --hash=sha256:80049883a4ae928fdcc661ba6803ec267665dc0e6f3ce2da91441079a6bb6387 ++ # via apache-beam ++google-cloud-dlp==3.34.0 \ ++ --hash=sha256:3a1a7fd335fd65641ac3cb3f24f96ee9345d546d413ad6c88071a59404b1a641 \ ++ --hash=sha256:6dfa3172520d5a7fa8ccce47a9622cde815f037b4aa6fb6d69984fd597bf8007 ++ # via apache-beam ++google-cloud-kms==3.11.0 \ ++ --hash=sha256:07f2829e4ed986220802d013219fe159ecbdecec35907a6ddeea37ea9daecd8d \ ++ --hash=sha256:5f7d7bdb347f13a8a2b7bad6cbdf3846a51690df7215586845b62851b88839f7 ++ # via apache-beam ++google-cloud-language==2.19.0 \ ++ --hash=sha256:3b88f6eabd1c2413a1c6c918cbe40a22a5d14401930309717dbb709b353c6c64 \ ++ --hash=sha256:a43044632c8aada30a9c3246e00bfc867a56188be0c6e08e8764731296a05e0b ++ # via apache-beam ++google-cloud-monitoring==2.29.1 \ ++ --hash=sha256:86cac55cdd2608561819d19544fb3c129bbb7dcecc445d8de426e34cd6fa8e49 \ ++ --hash=sha256:944a57031f20da38617d184d5658c1f938e019e8061f27fd944584831a1b9d5a ++ # via google-cloud-spanner ++google-cloud-pubsub==2.35.0 \ ++ --hash=sha256:2c0d1d7ccda52fa12fb73f34b7eb9899381e2fd931c7d47b10f724cdfac06f95 \ ++ --hash=sha256:c32e4eb29e532ec784b5abb5d674807715ec07895b7c022b9404871dec09970d ++ # via ++ # apache-beam ++ # google-cloud-pubsublite ++google-cloud-pubsublite==1.13.0 \ ++ --hash=sha256:00773be42f335ec0e76e0e3e6c72041c2795268433f48add29780cea41e8bd3e \ ++ --hash=sha256:cc56ca57755e7665a66f0c0025ca923f7bfeb39ba408859ffe87cb840c0e82b5 ++ # via apache-beam ++google-cloud-recommendations-ai==0.10.18 \ ++ --hash=sha256:a6bccb45744fd89f038aa3e19502d1f46ea61c438dd2c08528533f8e185ec469 \ ++ --hash=sha256:c5c4b569d8be96e65dc273d18a35e44147ef62f845c8a9e8afd93474802c60c8 ++ # via apache-beam ++google-cloud-resource-manager==1.16.0 \ ++ --hash=sha256:cc938f87cc36c2672f062b1e541650629e0d954c405a4dac35ceedee70c267c3 \ ++ --hash=sha256:fb9a2ad2b5053c508e1c407ac31abfd1a22e91c32876c1892830724195819a28 ++ # via google-cloud-aiplatform ++google-cloud-secret-manager==2.26.0 \ ++ --hash=sha256:0d1d6f76327685a0ed78a4cf50f289e1bfbbe56026ed0affa98663b86d6d50d6 \ ++ --hash=sha256:940a5447a6ec9951446fd1a0f22c81a4303fde164cd747aae152c5f5c8e6723e ++ # via apache-beam ++google-cloud-spanner==3.63.0 \ ++ --hash=sha256:6ffae0ed589bbbd2d8831495e266198f3d069005cfe65c664448c9a727c88e7b \ ++ --hash=sha256:e2a4fb3bdbad4688645f455d498705d3f935b7c9011f5c94c137b77569b47a62 ++ # via apache-beam ++google-cloud-storage==2.19.0 \ ++ --hash=sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba \ ++ --hash=sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++google-cloud-videointelligence==2.18.0 \ ++ --hash=sha256:2cf4a32f64f4e01fdfb78b7bf625aa82df9129c87854796348887eac60290e95 \ ++ --hash=sha256:b2ae39bd22d186218684a297c2fa2fa636e5874e69d39f719504d729f44639fd ++ # via apache-beam ++google-cloud-vision==3.12.1 \ ++ --hash=sha256:8c661bc0e7a6bd3d03a1a645b977af24ae3f21ccf3df8e213298659fd0d40813 \ ++ --hash=sha256:f99b83af7588d30e708b87e09ff73e43e380497fe82c799b9f05e03f310027c8 ++ # via apache-beam ++google-crc32c==1.8.0 \ ++ --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ ++ --hash=sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a \ ++ --hash=sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff \ ++ --hash=sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288 \ ++ --hash=sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411 \ ++ --hash=sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a \ ++ --hash=sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15 \ ++ --hash=sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb \ ++ --hash=sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa \ ++ --hash=sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962 \ ++ --hash=sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215 \ ++ --hash=sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b \ ++ --hash=sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27 \ ++ --hash=sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113 \ ++ --hash=sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f \ ++ --hash=sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f \ ++ --hash=sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d \ ++ --hash=sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2 \ ++ --hash=sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092 \ ++ --hash=sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7 \ ++ --hash=sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2 \ ++ --hash=sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93 \ ++ --hash=sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8 \ ++ --hash=sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21 \ ++ --hash=sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79 \ ++ --hash=sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2 \ ++ --hash=sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc \ ++ --hash=sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454 \ ++ --hash=sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2 \ ++ --hash=sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733 \ ++ --hash=sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697 \ ++ --hash=sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651 \ ++ --hash=sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c ++ # via ++ # google-cloud-bigtable ++ # google-cloud-storage ++ # google-resumable-media ++google-genai==1.66.0 \ ++ --hash=sha256:7f127a39cf695277104ce4091bb26e417c59bb46e952ff3699c3a982d9c474ee \ ++ --hash=sha256:ffc01647b65046bca6387320057aa51db0ad64bcc72c8e3e914062acfa5f7c49 ++ # via google-cloud-aiplatform ++google-resumable-media==2.8.0 \ ++ --hash=sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582 \ ++ --hash=sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae ++ # via ++ # google-cloud-bigquery ++ # google-cloud-storage ++googleapis-common-protos[grpc]==1.73.0 \ ++ --hash=sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a \ ++ --hash=sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8 ++ # via ++ # google-api-core ++ # grpc-google-iam-v1 ++ # grpcio-status ++grpc-google-iam-v1==0.14.3 \ ++ --hash=sha256:7a7f697e017a067206a3dfef44e4c634a34d3dee135fe7d7a4613fe3e59217e6 \ ++ --hash=sha256:879ac4ef33136c5491a6300e27575a9ec760f6cdf9a2518798c1b8977a5dc389 ++ # via ++ # google-cloud-bigtable ++ # google-cloud-kms ++ # google-cloud-pubsub ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++grpc-interceptor==0.15.4 \ ++ --hash=sha256:0035f33228693ed3767ee49d937bac424318db173fef4d2d0170b3215f254d9d \ ++ --hash=sha256:1f45c0bcb58b6f332f37c637632247c9b02bc6af0fdceb7ba7ce8d2ebbfb0926 ++ # via google-cloud-spanner ++grpcio==1.65.5 \ ++ --hash=sha256:05f02d68fc720e085f061b704ee653b181e6d5abfe315daef085719728d3d1fd \ ++ --hash=sha256:078038e150a897e5e402ed3d57f1d31ebf604cbed80f595bd281b5da40762a92 \ ++ --hash=sha256:0b2944390a496567de9e70418f3742b477d85d8ca065afa90432edc91b4bb8ad \ ++ --hash=sha256:11f8b16121768c1cb99d7dcb84e01510e60e6a206bf9123e134118802486f035 \ ++ --hash=sha256:1c4caafe71aef4dabf53274bbf4affd6df651e9f80beedd6b8e08ff438ed3260 \ ++ --hash=sha256:1cbc208edb9acf1cc339396a1a36b83796939be52f34e591c90292045b579fbf \ ++ --hash=sha256:238a625f391a1b9f5f069bdc5930f4fd71b74426bea52196fc7b83f51fa97d34 \ ++ --hash=sha256:2a6d8169812932feac514b420daffae8ab8e36f90f3122b94ae767e633296b17 \ ++ --hash=sha256:2b91ce647b6307f25650872454a4d02a2801f26a475f90d0b91ed8110baae589 \ ++ --hash=sha256:3207ae60d07e5282c134b6e02f9271a2cb523c6d7a346c6315211fe2bf8d61ed \ ++ --hash=sha256:32d60e18ff7c34fe3f6db3d35ad5c6dc99f5b43ff3982cb26fad4174462d10b1 \ ++ --hash=sha256:33158e56c6378063923c417e9fbdb28660b6e0e2835af42e67f5a7793f587af7 \ ++ --hash=sha256:47d0aaaab82823f0aa6adea5184350b46e2252e13a42a942db84da5b733f2e05 \ ++ --hash=sha256:55714ea852396ec9568f45f487639945ab674de83c12bea19d5ddbc3ae41ada3 \ ++ --hash=sha256:6c4e62bcf297a1568f627f39576dbfc27f1e5338a691c6dd5dd6b3979da51d1c \ ++ --hash=sha256:76991b7a6fb98630a3328839755181ce7c1aa2b1842aa085fd4198f0e5198960 \ ++ --hash=sha256:770bd4bd721961f6dd8049bc27338564ba8739913f77c0f381a9815e465ff965 \ ++ --hash=sha256:7a412959aa5f08c5ac04aa7b7c3c041f5e4298cadd4fcc2acff195b56d185ebc \ ++ --hash=sha256:84c901cdec16a092099f251ef3360d15e29ef59772150fa261d94573612539b5 \ ++ --hash=sha256:85ae8f8517d5bcc21fb07dbf791e94ed84cc28f84c903cdc2bd7eaeb437c8f45 \ ++ --hash=sha256:89c00a18801b1ed9cc441e29b521c354725d4af38c127981f2c950c796a09b6e \ ++ --hash=sha256:8da58ff80bc4556cf29bc03f5fff1f03b8387d6aaa7b852af9eb65b2cf833be4 \ ++ --hash=sha256:8e5c4c15ac3fe1eb68e46bc51e66ad29be887479f231f8237cf8416058bf0cc1 \ ++ --hash=sha256:a101696f9ece90a0829988ff72f1b1ea2358f3df035bdf6d675dd8b60c2c0894 \ ++ --hash=sha256:a2f80510f99f82d4eb825849c486df703f50652cea21c189eacc2b84f2bde764 \ ++ --hash=sha256:a70a20eed87bba647a38bedd93b3ce7db64b3f0e8e0952315237f7f5ca97b02d \ ++ --hash=sha256:a80e9a5e3f93c54f5eb82a3825ea1fc4965b2fa0026db2abfecb139a5c4ecdf1 \ ++ --hash=sha256:ab5ec837d8cee8dbce9ef6386125f119b231e4333cc6b6d57b6c5c7c82a72331 \ ++ --hash=sha256:b67d450f1e008fedcd81e097a3a400a711d8be1a8b20f852a7b8a73fead50fe3 \ ++ --hash=sha256:b7ca419f1462390851eec395b2089aad1e49546b52d4e2c972ceb76da69b10f8 \ ++ --hash=sha256:b8270b15b99781461b244f5c81d5c2bc9696ab9189fb5ff86c841417fb3b39fe \ ++ --hash=sha256:bc74f3f745c37e2c5685c9d2a2d5a94de00f286963f5213f763ae137bf4f2358 \ ++ --hash=sha256:c3655139d7be213c32c79ef6fb2367cae28e56ef68e39b1961c43214b457f257 \ ++ --hash=sha256:c97962720489ef31b5ad8a916e22bc31bba3664e063fb9f6702dce056d4aa61b \ ++ --hash=sha256:cabd706183ee08d8026a015af5819a0b3a8959bdc9d1f6fdacd1810f09200f2a \ ++ --hash=sha256:d3a9e35bcb045e39d7cac30464c285389b9a816ac2067e4884ad2c02e709ef8e \ ++ --hash=sha256:d750e9330eb14236ca11b78d0c494eed13d6a95eb55472298f0e547c165ee324 \ ++ --hash=sha256:d7df567b67d16d4177835a68d3f767bbcbad04da9dfb52cbd19171f430c898bd \ ++ --hash=sha256:ec6f219fb5d677a522b0deaf43cea6697b16f338cb68d009e30930c4aa0d2209 \ ++ --hash=sha256:ec71fc5b39821ad7d80db7473c8f8c2910f3382f0ddadfbcfc2c6c437107eb67 \ ++ --hash=sha256:ee6ed64a27588a2c94e8fa84fe8f3b5c89427d4d69c37690903d428ec61ca7e4 \ ++ --hash=sha256:f17f9fa2d947dbfaca01b3ab2c62eefa8240131fdc67b924eb42ce6032e3e5c1 \ ++ --hash=sha256:f5b5970341359341d0e4c789da7568264b2a89cd976c05ea476036852b5950cd \ ++ --hash=sha256:f79c87c114bf37adf408026b9e2e333fe9ff31dfc9648f6f80776c513145c813 \ ++ --hash=sha256:fa36dd8496d3af0d40165252a669fa4f6fd2db4b4026b9a9411cbf060b9d6a15 \ ++ --hash=sha256:fe6505376f5b00bb008e4e1418152e3ad3d954b629da286c7913ff3cfc0ff740 ++ # via ++ # apache-beam ++ # google-api-core ++ # google-cloud-bigquery-storage ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-videointelligence ++ # google-cloud-vision ++ # googleapis-common-protos ++ # grpc-google-iam-v1 ++ # grpc-interceptor ++ # grpcio-status ++grpcio-status==1.65.5 \ ++ --hash=sha256:2c9fa3af32efd26f01837d44305dce106973bc5357b9a9fc8bbd87bb8bf833d1 \ ++ --hash=sha256:44a445ce55375545a913e005be36fbec7999a4cc320d7aecb7a4469d3d49366c ++ # via ++ # google-api-core ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++grpclib==0.4.9 \ ++ --hash=sha256:7762ec1c8ed94dfad597475152dd35cbd11aecaaca2f243e29702435ca24cf0e \ ++ --hash=sha256:cc589c330fa81004c6400a52a566407574498cb5b055fa927013361e21466c46 ++ # via betterproto ++h11==0.16.0 \ ++ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ ++ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 ++ # via httpcore ++h2==4.3.0 \ ++ --hash=sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1 \ ++ --hash=sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd ++ # via grpclib ++hpack==4.1.0 \ ++ --hash=sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496 \ ++ --hash=sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca ++ # via h2 ++httpcore==1.0.9 \ ++ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ ++ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 ++ # via httpx ++httplib2==0.22.0 \ ++ --hash=sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc \ ++ --hash=sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81 ++ # via ++ # apache-beam ++ # google-apitools ++ # google-auth-httplib2 ++ # oauth2client ++httpx==0.28.1 \ ++ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ ++ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad ++ # via google-genai ++hyperframe==6.1.0 \ ++ --hash=sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5 \ ++ --hash=sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08 ++ # via h2 ++idna==3.11 \ ++ --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ ++ --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 ++ # via ++ # anyio ++ # httpx ++ # requests ++ # yarl ++importlib-metadata==8.7.1 \ ++ --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \ ++ --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151 ++ # via ++ # keyring ++ # opentelemetry-api ++jaraco-classes==3.4.0 \ ++ --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ ++ --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 ++ # via keyring ++jaraco-context==6.1.1 \ ++ --hash=sha256:0df6a0287258f3e364072c3e40d5411b20cafa30cb28c4839d24319cecf9f808 \ ++ --hash=sha256:bc046b2dc94f1e5532bd02402684414575cc11f565d929b6563125deb0a6e581 ++ # via keyring ++jaraco-functools==4.4.0 \ ++ --hash=sha256:9eec1e36f45c818d9bf307c8948eb03b2b56cd44087b3cdc989abca1f20b9176 \ ++ --hash=sha256:da21933b0417b89515562656547a77b4931f98176eb173644c0d35032a33d6bb ++ # via keyring ++jeepney==0.9.0 \ ++ --hash=sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683 \ ++ --hash=sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732 ++ # via ++ # keyring ++ # secretstorage ++jsonpickle==3.4.2 \ ++ --hash=sha256:2efa2778859b6397d5804b0a98d52cd2a7d9a70fcb873bc5a3ca5acca8f499ba \ ++ --hash=sha256:fd6c273278a02b3b66e3405db3dd2f4dbc8f4a4a3123bfcab3045177c6feb9c3 ++ # via apache-beam ++keyring==25.7.0 \ ++ --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f \ ++ --hash=sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b ++ # via keyrings-google-artifactregistry-auth ++keyrings-google-artifactregistry-auth==1.1.2 \ ++ --hash=sha256:bd6abb72740d2dfeb4a5c03c3b105c6f7dba169caa29dee3959694f1f02c77de \ ++ --hash=sha256:e3f18b50fa945c786593014dc225810d191671d4f5f8e12d9259e39bad3605a3 ++ # via apache-beam ++mmh3==5.2.1 \ ++ --hash=sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d \ ++ --hash=sha256:0634581290e6714c068f4aa24020acf7880927d1f0084fa753d9799ae9610082 \ ++ --hash=sha256:08043f7cb1fb9467c3fbbbaea7896986e7fbc81f4d3fd9289a73d9110ab6207a \ ++ --hash=sha256:0a3984146e414684a6be2862d84fcb1035f4984851cb81b26d933bab6119bf00 \ ++ --hash=sha256:0bbc17250b10d3466875a40a52520a6bac3c02334ca709207648abd3c223ed5c \ ++ --hash=sha256:0cc21533878e5586b80d74c281d7f8da7932bc8ace50b8d5f6dbf7e3935f63f1 \ ++ --hash=sha256:0d0b7e803191db5f714d264044e06189c8ccd3219e936cc184f07106bd17fd7b \ ++ --hash=sha256:113f78e7463a36dbbcea05bfe688efd7fa759d0f0c56e73c974d60dcfec3dfcc \ ++ --hash=sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104 \ ++ --hash=sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8 \ ++ --hash=sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9 \ ++ --hash=sha256:1d9f9a3ce559a5267014b04b82956993270f63ec91765e13e9fd73daf2d2738e \ ++ --hash=sha256:1e4ecee40ba19e6975e1120829796770325841c2f153c0e9aecca927194c6a2a \ ++ --hash=sha256:22b0f9971ec4e07e8223f2beebe96a6cfc779d940b6f27d26604040dd74d3a44 \ ++ --hash=sha256:26fb5b9c3946bf7f1daed7b37e0c03898a6f062149127570f8ede346390a0825 \ ++ --hash=sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4 \ ++ --hash=sha256:28cfab66577000b9505a0d068c731aee7ca85cd26d4d63881fab17857e0fe1fb \ ++ --hash=sha256:29bc3973676ae334412efdd367fcd11d036b7be3efc1ce2407ef8676dabfeb82 \ ++ --hash=sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f \ ++ --hash=sha256:2d5d542bf2abd0fd0361e8017d03f7cb5786214ceb4a40eef1539d6585d93386 \ ++ --hash=sha256:30e4d2084df019880d55f6f7bea35328d9b464ebee090baa372c096dc77556fb \ ++ --hash=sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d \ ++ --hash=sha256:368625fb01666655985391dbad3860dc0ba7c0d6b9125819f3121ee7292b4ac8 \ ++ --hash=sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593 \ ++ --hash=sha256:3a9fed49c6ce4ed7e73f13182760c65c816da006debe67f37635580dfb0fae00 \ ++ --hash=sha256:3c38d142c706201db5b2345166eeef1e7740e3e2422b470b8ba5c8727a9b4c7a \ ++ --hash=sha256:3cb61db880ec11e984348227b333259994c2c85caa775eb7875decb3768db890 \ ++ --hash=sha256:3d74a03fb57757ece25aa4b3c1c60157a1cece37a020542785f942e2f827eed5 \ ++ --hash=sha256:3f796b535008708846044c43302719c6956f39ca2d93f2edda5319e79a29efbb \ ++ --hash=sha256:41105377f6282e8297f182e393a79cfffd521dde37ace52b106373bdcd9ca5cb \ ++ --hash=sha256:41aac7002a749f08727cb91babff1daf8deac317c0b1f317adc69be0e6c375d1 \ ++ --hash=sha256:44983e45310ee5b9f73397350251cdf6e63a466406a105f1d16cb5baa659270b \ ++ --hash=sha256:4cbbde66f1183db040daede83dd86c06d663c5bb2af6de1142b7c8c37923dd74 \ ++ --hash=sha256:4eda76074cfca2787c8cf1bec603eaebdddd8b061ad5502f85cddae998d54f00 \ ++ --hash=sha256:4fc6cd65dc4d2fdb2625e288939a3566e36127a84811a4913f02f3d5931da52d \ ++ --hash=sha256:50885073e2909251d4718634a191c49ae5f527e5e1736d738e365c3e8be8f22b \ ++ --hash=sha256:5174a697ce042fa77c407e05efe41e03aa56dae9ec67388055820fb48cf4c3ba \ ++ --hash=sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b \ ++ --hash=sha256:54fe8518abe06a4c3852754bfd498b30cc58e667f376c513eac89a244ce781a4 \ ++ --hash=sha256:55dbbd8ffbc40d1697d5e2d0375b08599dae8746b0b08dea05eee4ce81648fac \ ++ --hash=sha256:57b52603e89355ff318025dd55158f6e71396c0f1f609d548e9ea9c94cc6ce0a \ ++ --hash=sha256:58370d05d033ee97224c81263af123dea3d931025030fd34b61227a768a8858a \ ++ --hash=sha256:5d87a3584093e1a89987e3d36d82c98d9621b2cb944e22a420aa1401e096758f \ ++ --hash=sha256:623f938f6a039536cc02b7582a07a080f13fdfd48f87e63201d92d7e34d09a18 \ ++ --hash=sha256:62815d2c67f2dd1be76a253d88af4e1da19aeaa1820146dec52cf8bee2958b16 \ ++ --hash=sha256:6290289fa5fb4c70fd7f72016e03633d60388185483ff3b162912c81205ae2cf \ ++ --hash=sha256:67e41a497bac88cc1de96eeba56eeb933c39d54bc227352f8455aa87c4ca4000 \ ++ --hash=sha256:6c85c38a279ca9295a69b9b088a2e48aa49737bb1b34e6a9dc6297c110e8d912 \ ++ --hash=sha256:6f01f044112d43a20be2f13a11683666d87151542ad627fe41a18b9791d2802f \ ++ --hash=sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5 \ ++ --hash=sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e \ ++ --hash=sha256:72d1cc63bcc91e14933f77d51b3df899d6a07d184ec515ea7f56bff659e124d7 \ ++ --hash=sha256:7374d6e3ef72afe49697ecd683f3da12f4fc06af2d75433d0580c6746d2fa025 \ ++ --hash=sha256:7501e9be34cb21e72fcfe672aafd0eee65c16ba2afa9dcb5500a587d3a0580f0 \ ++ --hash=sha256:76219cd1eefb9bf4af7856e3ae563d15158efa145c0aab01e9933051a1954045 \ ++ --hash=sha256:7aec798c2b01aaa65a55f1124f3405804184373abb318a3091325aece235f67c \ ++ --hash=sha256:7be6dfb49e48fd0a7d91ff758a2b51336f1cd21f9d44b20f6801f072bd080cdd \ ++ --hash=sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d \ ++ --hash=sha256:7e8ec5f606e0809426d2440e0683509fb605a8820a21ebd120dcdba61b74ef7f \ ++ --hash=sha256:7f196cd7910d71e9d9860da0ff7a77f64d22c1ad931f1dd18559a06e03109fc0 \ ++ --hash=sha256:82f3802bfc4751f420d591c5c864de538b71cea117fce67e4595c2afede08a15 \ ++ --hash=sha256:85ffc9920ffc39c5eee1e3ac9100c913a0973996fbad5111f939bbda49204bb7 \ ++ --hash=sha256:8e6c219e375f6341d0959af814296372d265a8ca1af63825f65e2e87c618f006 \ ++ --hash=sha256:8f767ba0911602ddef289404e33835a61168314ebd3c729833db2ed685824211 \ ++ --hash=sha256:8ff038d52ef6aa0f309feeba00c5095c9118d0abf787e8e8454d6048db2037fc \ ++ --hash=sha256:915e7a2418f10bd1151b1953df06d896db9783c9cfdb9a8ee1f9b3a4331ab503 \ ++ --hash=sha256:92883836caf50d5255be03d988d75bc93e3f86ba247b7ca137347c323f731deb \ ++ --hash=sha256:960b1b3efa39872ac8b6cc3a556edd6fb90ed74f08c9c45e028f1005b26aa55d \ ++ --hash=sha256:9aeaf53eaa075dd63e81512522fd180097312fb2c9f476333309184285c49ce0 \ ++ --hash=sha256:9d8089d853c7963a8ce87fff93e2a67075c0bc08684a08ea6ad13577c38ffc38 \ ++ --hash=sha256:a4130d0b9ce5fad6af07421b1aecc7e079519f70d6c05729ab871794eded8617 \ ++ --hash=sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f \ ++ --hash=sha256:add7ac388d1e0bf57259afbcf9ed05621a3bf11ce5ee337e7536f1e1aaf056b0 \ ++ --hash=sha256:b1f12bd684887a0a5d55e6363ca87056f361e45451105012d329b86ec19dbe0b \ ++ --hash=sha256:b3f99e1756fc48ad507b95e5d86f2fb21b3d495012ff13e6592ebac14033f166 \ ++ --hash=sha256:b4cce60d0223074803c9dbe0721ad3fa51dafe7d462fee4b656a1aa01ee07518 \ ++ --hash=sha256:baeb47635cb33375dee4924cd93d7f5dcaa786c740b08423b0209b824a1ee728 \ ++ --hash=sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad \ ++ --hash=sha256:bbfcb95d9a744e6e2827dfc66ad10e1020e0cac255eb7f85652832d5a264c2fc \ ++ --hash=sha256:bd6e7d363aa93bd3421b30b6af97064daf47bc96005bddba67c5ffbc6df426b8 \ ++ --hash=sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03 \ ++ --hash=sha256:c302245fd6c33d96bd169c7ccf2513c20f4c1e417c07ce9dce107c8bc3f8411f \ ++ --hash=sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2 \ ++ --hash=sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229 \ ++ --hash=sha256:cd471ede0d802dd936b6fab28188302b2d497f68436025857ca72cd3810423fe \ ++ --hash=sha256:d106493a60dcb4aef35a0fac85105e150a11cf8bc2b0d388f5a33272d756c966 \ ++ --hash=sha256:d30b650595fdbe32366b94cb14f30bb2b625e512bd4e1df00611f99dc5c27fd4 \ ++ --hash=sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6 \ ++ --hash=sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1 \ ++ --hash=sha256:d771f085fcdf4035786adfb1d8db026df1eb4b41dac1c3d070d1e49512843227 \ ++ --hash=sha256:dae0f0bd7d30c0ad61b9a504e8e272cb8391eed3f1587edf933f4f6b33437450 \ ++ --hash=sha256:db0562c5f71d18596dcd45e854cf2eeba27d7543e1a3acdafb7eef728f7fe85d \ ++ --hash=sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b \ ++ --hash=sha256:e080c0637aea036f35507e803a4778f119a9b436617694ae1c5c366805f1e997 \ ++ --hash=sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6 \ ++ --hash=sha256:e8b4b5580280b9265af3e0409974fb79c64cf7523632d03fbf11df18f8b0181e \ ++ --hash=sha256:e8b5378de2b139c3a830f0209c1e91f7705919a4b3e563a10955104f5097a70a \ ++ --hash=sha256:e904f2417f0d6f6d514f3f8b836416c360f306ddaee1f84de8eef1e722d212e5 \ ++ --hash=sha256:eee884572b06bbe8a2b54f424dbd996139442cf83c76478e1ec162512e0dd2c7 \ ++ --hash=sha256:f1fbb0a99125b1287c6d9747f937dc66621426836d1a2d50d05aecfc81911b57 \ ++ --hash=sha256:f40a95186a72fa0b67d15fef0f157bfcda00b4f59c8a07cbe5530d41ac35d105 \ ++ --hash=sha256:f6e0bfe77d238308839699944164b96a2eeccaf55f2af400f54dc20669d8d5f2 \ ++ --hash=sha256:f963eafc0a77a6c0562397da004f5876a9bcf7265a7bcc3205e29636bc4a1312 \ ++ --hash=sha256:fb9d44c25244e11c8be3f12c938ca8ba8404620ef8092245d2093c6ab3df260f \ ++ --hash=sha256:fc78739b5ec6e4fb02301984a3d442a91406e7700efbe305071e7fd1c78278f2 \ ++ --hash=sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a \ ++ --hash=sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b ++ # via google-cloud-spanner ++more-itertools==10.8.0 \ ++ --hash=sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b \ ++ --hash=sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd ++ # via ++ # jaraco-classes ++ # jaraco-functools ++multidict==6.7.1 \ ++ --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ ++ --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ ++ --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \ ++ --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \ ++ --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \ ++ --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \ ++ --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \ ++ --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \ ++ --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \ ++ --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \ ++ --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \ ++ --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \ ++ --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \ ++ --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \ ++ --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \ ++ --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \ ++ --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \ ++ --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \ ++ --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \ ++ --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \ ++ --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \ ++ --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \ ++ --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \ ++ --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \ ++ --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \ ++ --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \ ++ --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \ ++ --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \ ++ --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \ ++ --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \ ++ --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \ ++ --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \ ++ --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \ ++ --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \ ++ --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \ ++ --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \ ++ --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \ ++ --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \ ++ --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \ ++ --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \ ++ --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \ ++ --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \ ++ --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \ ++ --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \ ++ --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \ ++ --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \ ++ --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \ ++ --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \ ++ --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \ ++ --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \ ++ --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \ ++ --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \ ++ --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \ ++ --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \ ++ --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \ ++ --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \ ++ --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \ ++ --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \ ++ --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \ ++ --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \ ++ --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \ ++ --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \ ++ --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \ ++ --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \ ++ --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \ ++ --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \ ++ --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \ ++ --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \ ++ --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \ ++ --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \ ++ --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \ ++ --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \ ++ --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \ ++ --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \ ++ --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \ ++ --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \ ++ --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \ ++ --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \ ++ --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \ ++ --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \ ++ --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \ ++ --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \ ++ --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \ ++ --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \ ++ --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \ ++ --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \ ++ --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \ ++ --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \ ++ --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \ ++ --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \ ++ --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \ ++ --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \ ++ --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \ ++ --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \ ++ --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \ ++ --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \ ++ --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \ ++ --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \ ++ --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \ ++ --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \ ++ --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \ ++ --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \ ++ --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \ ++ --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \ ++ --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \ ++ --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \ ++ --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \ ++ --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \ ++ --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \ ++ --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \ ++ --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \ ++ --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \ ++ --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \ ++ --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \ ++ --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \ ++ --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \ ++ --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \ ++ --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \ ++ --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \ ++ --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \ ++ --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \ ++ --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \ ++ --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \ ++ --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \ ++ --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \ ++ --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \ ++ --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \ ++ --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \ ++ --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \ ++ --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \ ++ --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \ ++ --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \ ++ --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \ ++ --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \ ++ --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \ ++ --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \ ++ --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \ ++ --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \ ++ --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \ ++ --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \ ++ --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \ ++ --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \ ++ --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \ ++ --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \ ++ --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \ ++ --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19 ++ # via ++ # aiohttp ++ # grpclib ++ # yarl ++numpy==2.4.2 \ ++ --hash=sha256:00ab83c56211a1d7c07c25e3217ea6695e50a3e2f255053686b081dc0b091a82 \ ++ --hash=sha256:068cdb2d0d644cdb45670810894f6a0600797a69c05f1ac478e8d31670b8ee75 \ ++ --hash=sha256:0f01dcf33e73d80bd8dc0f20a71303abbafa26a19e23f6b68d1aa9990af90257 \ ++ --hash=sha256:0fece1d1f0a89c16b03442eae5c56dc0be0c7883b5d388e0c03f53019a4bfd71 \ ++ --hash=sha256:12e26134a0331d8dbd9351620f037ec470b7c75929cb8a1537f6bfe411152a1a \ ++ --hash=sha256:1ae241bbfc6ae276f94a170b14785e561cb5e7f626b6688cf076af4110887413 \ ++ --hash=sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181 \ ++ --hash=sha256:209fae046e62d0ce6435fcfe3b1a10537e858249b3d9b05829e2a05218296a85 \ ++ --hash=sha256:20abd069b9cda45874498b245c8015b18ace6de8546bf50dfa8cea1696ed06ef \ ++ --hash=sha256:21982668592194c609de53ba4933a7471880ccbaadcc52352694a59ecc860b3a \ ++ --hash=sha256:25f2059807faea4b077a2b6837391b5d830864b3543627f381821c646f31a63c \ ++ --hash=sha256:2653de5c24910e49c2b106499803124dde62a5a1fe0eedeaecf4309a5f639390 \ ++ --hash=sha256:2b8f157c8a6f20eb657e240f8985cc135598b2b46985c5bccbde7616dc9c6b1e \ ++ --hash=sha256:2fb882da679409066b4603579619341c6d6898fc83a8995199d5249f986e8e8f \ ++ --hash=sha256:40397bda92382fcec844066efb11f13e1c9a3e2a8e8f318fb72ed8b6db9f60f1 \ ++ --hash=sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b \ ++ --hash=sha256:47c5a6ed21d9452b10227e5e8a0e1c22979811cad7dcc19d8e3e2fb8fa03f1a3 \ ++ --hash=sha256:4f069069931240b3fc703f1e23df63443dbd6390614c8c44a87d96cd0ec81eb1 \ ++ --hash=sha256:52b913ec40ff7ae845687b0b34d8d93b60cb66dcee06996dd5c99f2fc9328657 \ ++ --hash=sha256:5633c0da313330fd20c484c78cdd3f9b175b55e1a766c4a174230c6b70ad8262 \ ++ --hash=sha256:5daf6f3914a733336dab21a05cdec343144600e964d2fcdabaac0c0269874b2a \ ++ --hash=sha256:5eea80d908b2c1f91486eb95b3fb6fab187e569ec9752ab7d9333d2e66bf2d6b \ ++ --hash=sha256:602f65afdef699cda27ec0b9224ae5dc43e328f4c24c689deaf77133dbee74d0 \ ++ --hash=sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae \ ++ --hash=sha256:66cb9422236317f9d44b67b4d18f44efe6e9c7f8794ac0462978513359461554 \ ++ --hash=sha256:6d82351358ffbcdcd7b686b90742a9b86632d6c1c051016484fa0b326a0a1548 \ ++ --hash=sha256:6e9f61981ace1360e42737e2bae58b27bf28a1b27e781721047d84bd754d32e7 \ ++ --hash=sha256:6ed0be1ee58eef41231a5c943d7d1375f093142702d5723ca2eb07db9b934b05 \ ++ --hash=sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1 \ ++ --hash=sha256:7df2de1e4fba69a51c06c28f5a3de36731eb9639feb8e1cf7e4a7b0daf4cf622 \ ++ --hash=sha256:7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1 \ ++ --hash=sha256:7f54844851cdb630ceb623dcec4db3240d1ac13d4990532446761baede94996a \ ++ --hash=sha256:805cc8de9fd6e7a22da5aed858e0ab16be5a4db6c873dde1d7451c541553aa27 \ ++ --hash=sha256:8906e71fd8afcb76580404e2a950caef2685df3d2a57fe82a86ac8d33cc007ba \ ++ --hash=sha256:89f7268c009bc492f506abd6f5265defa7cb3f7487dc21d357c3d290add45082 \ ++ --hash=sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443 \ ++ --hash=sha256:8e4549f8a3c6d13d55041925e912bfd834285ef1dd64d6bc7d542583355e2e98 \ ++ --hash=sha256:8e9afaeb0beff068b4d9cd20d322ba0ee1cecfb0b08db145e4ab4dd44a6b5110 \ ++ --hash=sha256:98f16a80e917003a12c0580f97b5f875853ebc33e2eaa4bccfc8201ac6869308 \ ++ --hash=sha256:9e35d3e0144137d9fdae62912e869136164534d64a169f86438bc9561b6ad49f \ ++ --hash=sha256:9e4424677ce4b47fe73c8b5556d876571f7c6945d264201180db2dc34f676ab5 \ ++ --hash=sha256:adb6ed2ad29b9e15321d167d152ee909ec73395901b70936f029c3bc6d7f4460 \ ++ --hash=sha256:aea4f66ff44dfddf8c2cffd66ba6538c5ec67d389285292fe428cb2c738c8aef \ ++ --hash=sha256:b21041e8cb6a1eb5312dd1d2f80a94d91efffb7a06b70597d44f1bd2dfc315ab \ ++ --hash=sha256:b2f0073ed0868db1dcd86e052d37279eef185b9c8db5bf61f30f46adac63c909 \ ++ --hash=sha256:b3a24467af63c67829bfaa61eecf18d5432d4f11992688537be59ecd6ad32f5e \ ++ --hash=sha256:b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695 \ ++ --hash=sha256:bba37bc29d4d85761deed3954a1bc62be7cf462b9510b51d367b769a8c8df325 \ ++ --hash=sha256:bd3a7a9f5847d2fb8c2c6d1c862fa109c31a9abeca1a3c2bd5a64572955b2979 \ ++ --hash=sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0 \ ++ --hash=sha256:c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32 \ ++ --hash=sha256:c3cd545784805de05aafe1dde61752ea49a359ccba9760c1e5d1c88a93bbf2b7 \ ++ --hash=sha256:c7ac672d699bf36275c035e16b65539931347d68b70667d28984c9fb34e07fa7 \ ++ --hash=sha256:cb7bbb88aa74908950d979eeaa24dbdf1a865e3c7e45ff0121d8f70387b55f73 \ ++ --hash=sha256:cd2bd2bbed13e213d6b55dc1d035a4f91748a7d3edc9480c13898b0353708920 \ ++ --hash=sha256:cda077c2e5b780200b6b3e09d0b42205a3d1c68f30c6dceb90401c13bff8fe74 \ ++ --hash=sha256:cf28c0c1d4c4bf00f509fa7eb02c58d7caf221b50b467bcb0d9bbf1584d5c821 \ ++ --hash=sha256:d0d9b7c93578baafcbc5f0b83eaf17b79d345c6f36917ba0c67f45226911d499 \ ++ --hash=sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000 \ ++ --hash=sha256:d30291931c915b2ab5717c2974bb95ee891a1cf22ebc16a8006bd59cd210d40a \ ++ --hash=sha256:d9f64d786b3b1dd742c946c42d15b07497ed14af1a1f3ce840cce27daa0ce913 \ ++ --hash=sha256:da6cad4e82cb893db4b69105c604d805e0c3ce11501a55b5e9f9083b47d2ffe8 \ ++ --hash=sha256:df1b10187212b198dd45fa943d8985a3c8cf854aed4923796e0e019e113a1bda \ ++ --hash=sha256:e04ae107ac591763a47398bb45b568fc38f02dbc4aa44c063f67a131f99346cb \ ++ --hash=sha256:e6dee3bb76aa4009d5a912180bf5b2de012532998d094acee25d9cb8dee3e44a \ ++ --hash=sha256:e7e88598032542bd49af7c4747541422884219056c268823ef6e5e89851c8825 \ ++ --hash=sha256:e98c97502435b53741540a5717a6749ac2ada901056c7db951d33e11c885cc7d \ ++ --hash=sha256:ec055f6dae239a6299cace477b479cca2fc125c5675482daf1dd886933a1076f \ ++ --hash=sha256:f74f0f7779cc7ae07d1810aab8ac6b1464c3eafb9e283a40da7309d5e6e48fbb \ ++ --hash=sha256:fbde1b0c6e81d56f5dccd95dd4a711d9b95df1ae4009a60887e56b27e8d903fa \ ++ --hash=sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236 \ ++ --hash=sha256:fd49860271d52127d61197bb50b64f58454e9f578cb4b2c001a6de8b1f50b0b1 ++ # via apache-beam ++oauth2client==4.1.3 \ ++ --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ ++ --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 ++ # via google-apitools ++objsize==0.7.1 \ ++ --hash=sha256:634a0c134c4b1ff2c340fe29caf58bc0a16cb2ff7c556df609d04f026fdf4eca \ ++ --hash=sha256:91e68d2a3031efb61b0e8cb7f995ddaeb65fe5ace9e737785e029f0932c2e619 ++ # via apache-beam ++opentelemetry-api==1.40.0 \ ++ --hash=sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f \ ++ --hash=sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9 ++ # via ++ # google-cloud-pubsub ++ # google-cloud-spanner ++ # opentelemetry-resourcedetector-gcp ++ # opentelemetry-sdk ++ # opentelemetry-semantic-conventions ++opentelemetry-resourcedetector-gcp==1.11.0a0 \ ++ --hash=sha256:5d65a2a039b1d40c6f41421dbb08d5f441368275ac6de6e76a8fccd1f6acb67e \ ++ --hash=sha256:915a1d6fd15daca9eedd3fc52b0f705375054f2ef140e2e7a6b4cca95a47cdb1 ++ # via google-cloud-spanner ++opentelemetry-sdk==1.40.0 \ ++ --hash=sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2 \ ++ --hash=sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1 ++ # via ++ # google-cloud-pubsub ++ # google-cloud-spanner ++ # opentelemetry-resourcedetector-gcp ++opentelemetry-semantic-conventions==0.61b0 \ ++ --hash=sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a \ ++ --hash=sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2 ++ # via ++ # google-cloud-spanner ++ # opentelemetry-sdk ++orjson==3.11.7 \ ++ --hash=sha256:043d3006b7d32c7e233b8cfb1f01c651013ea079e08dcef7189a29abd8befe11 \ ++ --hash=sha256:0527a4510c300e3b406591b0ba69b5dc50031895b0a93743526a3fc45f59d26e \ ++ --hash=sha256:0724e265bc548af1dedebd9cb3d24b4e1c1e685a343be43e87ba922a5c5fff2f \ ++ --hash=sha256:136dcd6a2e796dfd9ffca9fc027d778567b0b7c9968d092842d3c323cef88aa8 \ ++ --hash=sha256:14f440c7268c8f8633d1b3d443a434bd70cb15686117ea6beff8fdc8f5917a1e \ ++ --hash=sha256:1d98b30cc1313d52d4af17d9c3d307b08389752ec5f2e5febdfada70b0f8c733 \ ++ --hash=sha256:1eb80451a9c351a71dfaf5b7ccc13ad065405217726b59fdbeadbcc544f9d223 \ ++ --hash=sha256:1ee5cc7160a821dfe14f130bc8e63e7611051f964b463d9e2a3a573204446a4d \ ++ --hash=sha256:23d6c20517a97a9daf1d48b580fcdc6f0516c6f4b5038823426033690b4d2650 \ ++ --hash=sha256:26c3b9132f783b7d7903bf1efb095fed8d4a3a85ec0d334ee8beff3d7a4749d5 \ ++ --hash=sha256:31c80ce534ac4ea3739c5ee751270646cbc46e45aea7576a38ffec040b4029a1 \ ++ --hash=sha256:3726be79e36e526e3d9c1aceaadbfb4a04ee80a72ab47b3f3c17fefb9812e7b8 \ ++ --hash=sha256:390a1dce0c055ddf8adb6aa94a73b45a4a7d7177b5c584b8d1c1947f2ba60fb3 \ ++ --hash=sha256:3a2479753bbb95b0ebcf7969f562cdb9668e6d12416a35b0dda79febf89cdea2 \ ++ --hash=sha256:3c4bc6c6ac52cdaa267552544c73e486fecbd710b7ac09bc024d5a78555a22f6 \ ++ --hash=sha256:411ebaf34d735e25e358a6d9e7978954a9c9d58cfb47bc6683cdc3964cd2f910 \ ++ --hash=sha256:4682d1db3bcebd2b64757e0ddf9e87ae5f00d29d16c5cdf3a62f561d08cc3dd2 \ ++ --hash=sha256:4a2e9c5be347b937a2e0203866f12bba36082e89b402ddb9e927d5822e43088d \ ++ --hash=sha256:57036b27ac8a25d81112eb0cc9835cd4833c5b16e1467816adc0015f59e870dc \ ++ --hash=sha256:5ede977b5fe5ac91b1dffc0a517ca4542d2ec8a6a4ff7b2652d94f640796342a \ ++ --hash=sha256:5fdfad2093bdd08245f2e204d977facd5f871c88c4a71230d5bcbd0e43bf6222 \ ++ --hash=sha256:623ad1b9548ef63886319c16fa317848e465a21513b31a6ad7b57443c3e0dcf5 \ ++ --hash=sha256:652c6c3af76716f4a9c290371ba2e390ede06f6603edb277b481daf37f6f464e \ ++ --hash=sha256:6543001328aa857187f905308a028935864aefe9968af3848401b6fe80dbb471 \ ++ --hash=sha256:6e776b998ac37c0396093d10290e60283f59cfe0fc3fccbd0ccc4bd04dd19892 \ ++ --hash=sha256:71924496986275a737f38e3f22b4e0878882b3f7a310d2ff4dc96e812789120c \ ++ --hash=sha256:733ae23ada68b804b222c44affed76b39e30806d38660bf1eb200520d259cc16 \ ++ --hash=sha256:7477aa6a6ec6139c5cb1cc7b214643592169a5494d200397c7fc95d740d5fcf3 \ ++ --hash=sha256:79cacb0b52f6004caf92405a7e1f11e6e2de8bdf9019e4f76b44ba045125cd6b \ ++ --hash=sha256:7ba61079379b0ae29e117db13bda5f28d939766e410d321ec1624afc6a0b0504 \ ++ --hash=sha256:800988273a014a0541483dc81021247d7eacb0c845a9d1a34a422bc718f41539 \ ++ --hash=sha256:814be4b49b228cfc0b3c565acf642dd7d13538f966e3ccde61f4f55be3e20785 \ ++ --hash=sha256:845c3e0d8ded9c9271cd79596b9b552448b885b97110f628fb687aee2eed11c1 \ ++ --hash=sha256:849e38203e5be40b776ed2718e587faf204d184fc9a008ae441f9442320c0cab \ ++ --hash=sha256:89e13dd3f89f1c38a9c9eba5fbf7cdc2d1feca82f5f290864b4b7a6aac704576 \ ++ --hash=sha256:89e440ebc74ce8ab5c7bc4ce6757b4a6b1041becb127df818f6997b5c71aa60b \ ++ --hash=sha256:8ff206156006da5b847c9304b6308a01e8cdbc8cce824e2779a5ba71c3def141 \ ++ --hash=sha256:91c81ef070c8f3220054115e1ef468b1c9ce8497b4e526cb9f68ab4dc0a7ac62 \ ++ --hash=sha256:9487abc2c2086e7c8eb9a211d2ce8855bae0e92586279d0d27b341d5ad76c85c \ ++ --hash=sha256:962d046ee1765f74a1da723f4b33e3b228fe3a48bd307acce5021dfefe0e29b2 \ ++ --hash=sha256:996b65230271f1a97026fd0e6a753f51fbc0c335d2ad0c6201f711b0da32693b \ ++ --hash=sha256:9b1a67243945819ce55d24a30b59d6a168e86220452d2c96f4d1f093e71c0c49 \ ++ --hash=sha256:9c0b51672e466fd7e56230ffbae7f1639e18d0ce023351fb75da21b71bc2c960 \ ++ --hash=sha256:9e54f3808e2b6b945078c41aa8d9b5834b28c50843846e97807e5adb75fa9705 \ ++ --hash=sha256:a02c833f38f36546ba65a452127633afce4cf0dd7296b753d3bb54e55e5c0174 \ ++ --hash=sha256:a12b80df61aab7b98b490fe9e4879925ba666fccdfcd175252ce4d9035865ace \ ++ --hash=sha256:a16bcd08ab0bcdfc7e8801d9c4a9cc17e58418e4d48ddc6ded4e9e4b1a94062b \ ++ --hash=sha256:a56df3239294ea5964adf074c54bcc4f0ccd21636049a2cf3ca9cf03b5d03cf1 \ ++ --hash=sha256:a709e881723c9b18acddcfb8ba357322491ad553e277cf467e1e7e20e2d90561 \ ++ --hash=sha256:ab49d4b2a6a1d415ddb9f37a21e02e0d5dbfe10b7870b21bf779fc21e9156157 \ ++ --hash=sha256:ae9e0b37a834cef7ce8f99de6498f8fad4a2c0bf6bfc3d02abd8ed56aa15b2de \ ++ --hash=sha256:b4a9eefdc70bf8bf9857f0290f973dec534ac84c35cd6a7f4083be43e7170a8f \ ++ --hash=sha256:b63c6e6738d7c3470ad01601e23376aa511e50e1f3931395b9f9c722406d1a67 \ ++ --hash=sha256:b7b1dae39230a393df353827c855a5f176271c23434cfd2db74e0e424e693e10 \ ++ --hash=sha256:b8d14b71c0b12963fe8a62aac87119f1afdf4cb88a400f61ca5ae581449efcb5 \ ++ --hash=sha256:b9f95dcdea9d4f805daa9ddf02617a89e484c6985fa03055459f90e87d7a0757 \ ++ --hash=sha256:b9fc4d0f81f394689e0814617aadc4f2ea0e8025f38c226cbf22d3b5ddbf025d \ ++ --hash=sha256:bd03ea7606833655048dab1a00734a2875e3e86c276e1d772b2a02556f0d895f \ ++ --hash=sha256:bd0d68edd7dfca1b2eca9361a44ac9f24b078de3481003159929a0573f21a6bf \ ++ --hash=sha256:bda117c4148e81f746655d5a3239ae9bd00cb7bc3ca178b5fc5a5997e9744183 \ ++ --hash=sha256:bf742e149121dc5648ba0a08ea0871e87b660467ef168a3a5e53bc1fbd64bb74 \ ++ --hash=sha256:c2428d358d85e8da9d37cba18b8c4047c55222007a84f97156a5b22028dfbfc0 \ ++ --hash=sha256:c2e85fe4698b6a56d5e2ebf7ae87544d668eb6bde1ad1226c13f44663f20ec9e \ ++ --hash=sha256:c43b8b5bab288b6b90dac410cca7e986a4fa747a2e8f94615aea407da706980d \ ++ --hash=sha256:cededd6738e1c153530793998e31c05086582b08315db48ab66649768f326baa \ ++ --hash=sha256:d06e5c5fed5caedd2e540d62e5b1c25e8c82431b9e577c33537e5fa4aa909539 \ ++ --hash=sha256:d772afdb22555f0c58cfc741bdae44180122b3616faa1ecadb595cd526e4c993 \ ++ --hash=sha256:d897e81f8d0cbd2abb82226d1860ad2e1ab3ff16d7b08c96ca00df9d45409ef4 \ ++ --hash=sha256:de0a37f21d0d364954ad5de1970491d7fbd0fb1ef7417d4d56a36dc01ba0c0a0 \ ++ --hash=sha256:e7745312efa9e11c17fbd3cb3097262d079da26930ae9ae7ba28fb738367cbad \ ++ --hash=sha256:ed46f17096e28fb28d2975834836a639af7278aa87c84f68ab08fbe5b8bd75fa \ ++ --hash=sha256:f4f7c956b5215d949a1f65334cf9d7612dde38f20a95f2315deef167def91a6f \ ++ --hash=sha256:f50979824bde13d32b4320eedd513431c921102796d86be3eee0b58e58a3ecd1 \ ++ --hash=sha256:f904c24bdeabd4298f7a977ef14ca2a022ca921ed670b92ecd16ab6f3d01f867 ++ # via apache-beam ++overrides==7.7.0 \ ++ --hash=sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a \ ++ --hash=sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49 ++ # via google-cloud-pubsublite ++packaging==26.0 \ ++ --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ ++ --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++ # google-cloud-bigquery ++pg8000==1.31.5 \ ++ --hash=sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201 \ ++ --hash=sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78 ++ # via apache-beam ++pluggy==1.6.0 \ ++ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ ++ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 ++ # via keyrings-google-artifactregistry-auth ++propcache==0.4.1 \ ++ --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e \ ++ --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 \ ++ --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be \ ++ --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 \ ++ --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 \ ++ --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b \ ++ --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 \ ++ --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf \ ++ --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 \ ++ --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 \ ++ --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 \ ++ --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 \ ++ --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 \ ++ --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 \ ++ --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 \ ++ --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 \ ++ --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 \ ++ --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc \ ++ --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe \ ++ --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb \ ++ --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 \ ++ --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 \ ++ --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e \ ++ --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff \ ++ --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 \ ++ --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 \ ++ --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 \ ++ --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 \ ++ --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf \ ++ --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 \ ++ --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a \ ++ --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc \ ++ --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a \ ++ --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 \ ++ --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 \ ++ --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 \ ++ --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 \ ++ --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 \ ++ --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 \ ++ --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af \ ++ --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa \ ++ --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 \ ++ --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc \ ++ --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 \ ++ --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 \ ++ --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf \ ++ --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 \ ++ --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 \ ++ --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c \ ++ --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 \ ++ --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e \ ++ --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 \ ++ --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 \ ++ --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 \ ++ --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 \ ++ --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db \ ++ --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f \ ++ --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d \ ++ --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 \ ++ --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e \ ++ --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 \ ++ --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 \ ++ --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a \ ++ --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b \ ++ --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f \ ++ --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f \ ++ --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 \ ++ --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 \ ++ --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 \ ++ --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 \ ++ --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db \ ++ --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded \ ++ --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb \ ++ --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 \ ++ --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 \ ++ --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 \ ++ --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 \ ++ --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 \ ++ --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f \ ++ --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b \ ++ --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 \ ++ --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 \ ++ --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 \ ++ --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c \ ++ --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 \ ++ --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 \ ++ --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 \ ++ --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 \ ++ --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 \ ++ --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 \ ++ --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db \ ++ --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac \ ++ --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 \ ++ --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 \ ++ --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f \ ++ --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 \ ++ --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 \ ++ --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 \ ++ --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 \ ++ --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a \ ++ --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f \ ++ --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 \ ++ --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 \ ++ --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 \ ++ --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 \ ++ --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 \ ++ --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 \ ++ --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 \ ++ --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 \ ++ --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 \ ++ --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e \ ++ --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 \ ++ --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d \ ++ --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 \ ++ --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c \ ++ --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e \ ++ --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 \ ++ --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af \ ++ --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f \ ++ --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 \ ++ --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 \ ++ --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 ++ # via ++ # aiohttp ++ # yarl ++proto-plus==1.27.1 \ ++ --hash=sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147 \ ++ --hash=sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc ++ # via ++ # apache-beam ++ # google-api-core ++ # google-cloud-aiplatform ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++ # google-cloud-videointelligence ++ # google-cloud-vision ++protobuf==5.29.6 \ ++ --hash=sha256:36ade6ff88212e91aef4e687a971a11d7d24d6948a66751abc1b3238648f5d05 \ ++ --hash=sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1 \ ++ --hash=sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86 \ ++ --hash=sha256:76e07e6567f8baf827137e8d5b8204b6c7b6488bbbff1bf0a72b383f77999c18 \ ++ --hash=sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda \ ++ --hash=sha256:831e2da16b6cc9d8f1654c041dd594eda43391affd3c03a91bea7f7f6da106d6 \ ++ --hash=sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6 \ ++ --hash=sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269 \ ++ --hash=sha256:cb4c86de9cd8a7f3a256b9744220d87b847371c6b2f10bde87768918ef33ba49 \ ++ --hash=sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723 \ ++ --hash=sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9 ++ # via ++ # apache-beam ++ # google-api-core ++ # google-cloud-aiplatform ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++ # google-cloud-videointelligence ++ # google-cloud-vision ++ # googleapis-common-protos ++ # grpc-google-iam-v1 ++ # grpcio-status ++ # proto-plus ++pyarrow==18.1.0 \ ++ --hash=sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe \ ++ --hash=sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e \ ++ --hash=sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54 \ ++ --hash=sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99 \ ++ --hash=sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e \ ++ --hash=sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9 \ ++ --hash=sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181 \ ++ --hash=sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76 \ ++ --hash=sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c \ ++ --hash=sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c \ ++ --hash=sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56 \ ++ --hash=sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754 \ ++ --hash=sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b \ ++ --hash=sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9 \ ++ --hash=sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992 \ ++ --hash=sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc \ ++ --hash=sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7 \ ++ --hash=sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa \ ++ --hash=sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b \ ++ --hash=sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73 \ ++ --hash=sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812 \ ++ --hash=sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d \ ++ --hash=sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052 \ ++ --hash=sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191 \ ++ --hash=sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386 \ ++ --hash=sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324 \ ++ --hash=sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4 \ ++ --hash=sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba \ ++ --hash=sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470 \ ++ --hash=sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71 \ ++ --hash=sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30 \ ++ --hash=sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33 \ ++ --hash=sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a \ ++ --hash=sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8 \ ++ --hash=sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee \ ++ --hash=sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c \ ++ --hash=sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6 \ ++ --hash=sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854 \ ++ --hash=sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0 \ ++ --hash=sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21 \ ++ --hash=sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2 \ ++ --hash=sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c ++ # via apache-beam ++pyarrow-hotfix==0.7 \ ++ --hash=sha256:3236f3b5f1260f0e2ac070a55c1a7b339c4bb7267839bd2015e283234e758100 \ ++ --hash=sha256:59399cd58bdd978b2e42816a4183a55c6472d4e33d183351b6069f11ed42661d ++ # via apache-beam ++pyasn1==0.6.2 \ ++ --hash=sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf \ ++ --hash=sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b ++ # via ++ # oauth2client ++ # pyasn1-modules ++ # rsa ++pyasn1-modules==0.4.2 \ ++ --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ ++ --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 ++ # via ++ # google-auth ++ # oauth2client ++pycparser==3.0 \ ++ --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ ++ --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 ++ # via cffi ++pydantic==2.12.5 \ ++ --hash=sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49 \ ++ --hash=sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d ++ # via ++ # google-cloud-aiplatform ++ # google-genai ++pydantic-core==2.41.5 \ ++ --hash=sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90 \ ++ --hash=sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740 \ ++ --hash=sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504 \ ++ --hash=sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84 \ ++ --hash=sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33 \ ++ --hash=sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c \ ++ --hash=sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0 \ ++ --hash=sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e \ ++ --hash=sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0 \ ++ --hash=sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a \ ++ --hash=sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34 \ ++ --hash=sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2 \ ++ --hash=sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3 \ ++ --hash=sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815 \ ++ --hash=sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14 \ ++ --hash=sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba \ ++ --hash=sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375 \ ++ --hash=sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf \ ++ --hash=sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963 \ ++ --hash=sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1 \ ++ --hash=sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808 \ ++ --hash=sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553 \ ++ --hash=sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1 \ ++ --hash=sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2 \ ++ --hash=sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5 \ ++ --hash=sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470 \ ++ --hash=sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2 \ ++ --hash=sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b \ ++ --hash=sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660 \ ++ --hash=sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c \ ++ --hash=sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093 \ ++ --hash=sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5 \ ++ --hash=sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594 \ ++ --hash=sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008 \ ++ --hash=sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a \ ++ --hash=sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a \ ++ --hash=sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd \ ++ --hash=sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284 \ ++ --hash=sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586 \ ++ --hash=sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869 \ ++ --hash=sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294 \ ++ --hash=sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f \ ++ --hash=sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66 \ ++ --hash=sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51 \ ++ --hash=sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc \ ++ --hash=sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97 \ ++ --hash=sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a \ ++ --hash=sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d \ ++ --hash=sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9 \ ++ --hash=sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c \ ++ --hash=sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07 \ ++ --hash=sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36 \ ++ --hash=sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e \ ++ --hash=sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05 \ ++ --hash=sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e \ ++ --hash=sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941 \ ++ --hash=sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3 \ ++ --hash=sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612 \ ++ --hash=sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3 \ ++ --hash=sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b \ ++ --hash=sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe \ ++ --hash=sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146 \ ++ --hash=sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11 \ ++ --hash=sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60 \ ++ --hash=sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd \ ++ --hash=sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b \ ++ --hash=sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c \ ++ --hash=sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a \ ++ --hash=sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460 \ ++ --hash=sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1 \ ++ --hash=sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf \ ++ --hash=sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf \ ++ --hash=sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858 \ ++ --hash=sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2 \ ++ --hash=sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9 \ ++ --hash=sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2 \ ++ --hash=sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3 \ ++ --hash=sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6 \ ++ --hash=sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770 \ ++ --hash=sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d \ ++ --hash=sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc \ ++ --hash=sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23 \ ++ --hash=sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26 \ ++ --hash=sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa \ ++ --hash=sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8 \ ++ --hash=sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d \ ++ --hash=sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3 \ ++ --hash=sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d \ ++ --hash=sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034 \ ++ --hash=sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9 \ ++ --hash=sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1 \ ++ --hash=sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56 \ ++ --hash=sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b \ ++ --hash=sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c \ ++ --hash=sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a \ ++ --hash=sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e \ ++ --hash=sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9 \ ++ --hash=sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5 \ ++ --hash=sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a \ ++ --hash=sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556 \ ++ --hash=sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e \ ++ --hash=sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49 \ ++ --hash=sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2 \ ++ --hash=sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9 \ ++ --hash=sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b \ ++ --hash=sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc \ ++ --hash=sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb \ ++ --hash=sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0 \ ++ --hash=sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8 \ ++ --hash=sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82 \ ++ --hash=sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69 \ ++ --hash=sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b \ ++ --hash=sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c \ ++ --hash=sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75 \ ++ --hash=sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5 \ ++ --hash=sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f \ ++ --hash=sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad \ ++ --hash=sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b \ ++ --hash=sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7 \ ++ --hash=sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425 \ ++ --hash=sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52 ++ # via pydantic ++pymongo==4.16.0 \ ++ --hash=sha256:03f42396c1b2c6f46f5401c5b185adc25f6113716e16d9503977ee5386fca0fb \ ++ --hash=sha256:12762e7cc0f8374a8cae3b9f9ed8dabb5d438c7b33329232dd9b7de783454033 \ ++ --hash=sha256:15bb062c0d6d4b0be650410032152de656a2a9a2aa4e1a7443a22695afacb103 \ ++ --hash=sha256:19a1c96e7f39c7a59a9cfd4d17920cf9382f6f684faeff4649bf587dc59f8edc \ ++ --hash=sha256:1c01e8a7cd0ea66baf64a118005535ab5bf9f9eb63a1b50ac3935dccf9a54abe \ ++ --hash=sha256:1d638b0b1b294d95d0fdc73688a3b61e05cc4188872818cd240d51460ccabcb5 \ ++ --hash=sha256:21d02cc10a158daa20cb040985e280e7e439832fc6b7857bff3d53ef6914ad50 \ ++ --hash=sha256:2290909275c9b8f637b0a92eb9b89281e18a72922749ebb903403ab6cc7da914 \ ++ --hash=sha256:25a6b03a68f9907ea6ec8bc7cf4c58a1b51a18e23394f962a6402f8e46d41211 \ ++ --hash=sha256:2a3ba6be3d8acf64b77cdcd4e36f0e4a8e87965f14a8b09b90ca86f10a1dd2f2 \ ++ --hash=sha256:2b0714d7764efb29bf9d3c51c964aed7c4c7237b341f9346f15ceaf8321fdb35 \ ++ --hash=sha256:2cd60cd1e05de7f01927f8e25ca26b3ea2c09de8723241e5d3bcfdc70eaff76b \ ++ --hash=sha256:2d0082631a7510318befc2b4fdab140481eb4b9dd62d9245e042157085da2a70 \ ++ --hash=sha256:311d4549d6bf1f8c61d025965aebb5ba29d1481dc6471693ab91610aaffbc0eb \ ++ --hash=sha256:36ef2fee50eee669587d742fb456e349634b4fcf8926208766078b089054b24b \ ++ --hash=sha256:3ead8a0050c53eaa55935895d6919d393d0328ec24b2b9115bdbe881aa222673 \ ++ --hash=sha256:46ffb728d92dd5b09fc034ed91acf5595657c7ca17d4cf3751322cd554153c17 \ ++ --hash=sha256:4a19ea46a0fe71248965305a020bc076a163311aefbaa1d83e47d06fa30ac747 \ ++ --hash=sha256:4a9390dce61d705a88218f0d7b54d7e1fa1b421da8129fc7c009e029a9a6b81e \ ++ --hash=sha256:4c4872299ebe315a79f7f922051061634a64fda95b6b17677ba57ef00b2ba2a4 \ ++ --hash=sha256:4cd047ba6cc83cc24193b9208c93e134a985ead556183077678c59af7aacc725 \ ++ --hash=sha256:4d4f7ba040f72a9f43a44059872af5a8c8c660aa5d7f90d5344f2ed1c3c02721 \ ++ --hash=sha256:4d79aa147ce86aef03079096d83239580006ffb684eead593917186aee407767 \ ++ --hash=sha256:4fbb8d3552c2ad99d9e236003c0b5f96d5f05e29386ba7abae73949bfebc13dd \ ++ --hash=sha256:55f8d5a6fe2fa0b823674db2293f92d74cd5f970bc0360f409a1fc21003862d3 \ ++ --hash=sha256:5b9c6d689bbe5beb156374508133218610e14f8c81e35bc17d7a14e30ab593e6 \ ++ --hash=sha256:5d9fdb386cf958e6ef6ff537d6149be7edb76c3268cd6833e6c36aa447e4443f \ ++ --hash=sha256:60307bb91e0ab44e560fe3a211087748b2b5f3e31f403baf41f5b7b0a70bd104 \ ++ --hash=sha256:61567f712bda04c7545a037e3284b4367cad8d29b3dec84b4bf3b2147020a75b \ ++ --hash=sha256:66af44ed23686dd5422307619a6db4b56733c5e36fe8c4adf91326dcf993a043 \ ++ --hash=sha256:6af1aaa26f0835175d2200e62205b78e7ec3ffa430682e322cc91aaa1a0dbf28 \ ++ --hash=sha256:6b2a20edb5452ac8daa395890eeb076c570790dfce6b7a44d788af74c2f8cf96 \ ++ --hash=sha256:6f2077ec24e2f1248f9cac7b9a2dfb894e50cc7939fcebfb1759f99304caabef \ ++ --hash=sha256:77cfd37a43a53b02b7bd930457c7994c924ad8bbe8dff91817904bcbf291b371 \ ++ --hash=sha256:78037d02389745e247fe5ab0bcad5d1ab30726eaac3ad79219c7d6bbb07eec53 \ ++ --hash=sha256:7902882ed0efb7f0e991458ab3b8cf0eb052957264949ece2f09b63c58b04f78 \ ++ --hash=sha256:85dc2f3444c346ea019a371e321ac868a4fab513b7a55fe368f0cc78de8177cc \ ++ --hash=sha256:8a0f73af1ea56c422b2dcfc0437459148a799ef4231c6aee189d2d4c59d6728f \ ++ --hash=sha256:8a254d49a9ffe9d7f888e3c677eed3729b14ce85abb08cd74732cead6ccc3c66 \ ++ --hash=sha256:8ba8405065f6e258a6f872fe62d797a28f383a12178c7153c01ed04e845c600c \ ++ --hash=sha256:91899dd7fb9a8c50f09c3c1cf0cb73bfbe2737f511f641f19b9650deb61c00ca \ ++ --hash=sha256:91ac0cb0fe2bf17616c2039dac88d7c9a5088f5cb5829b27c9d250e053664d31 \ ++ --hash=sha256:92a232af9927710de08a6c16a9710cc1b175fb9179c0d946cd4e213b92b2a69a \ ++ --hash=sha256:948152b30eddeae8355495f9943a3bf66b708295c0b9b6f467de1c620f215487 \ ++ --hash=sha256:96aa7ab896889bf330209d26459e493d00f8855772a9453bfb4520bb1f495baf \ ++ --hash=sha256:9caacac0dd105e2555521002e2d17afc08665187017b466b5753e84c016628e6 \ ++ --hash=sha256:9d9885aad05f82fd7ea0c9ca505d60939746b39263fa273d0125170da8f59098 \ ++ --hash=sha256:9dc2c00bed568732b89e211b6adca389053d5e6d2d5a8979e80b813c3ec4d1f9 \ ++ --hash=sha256:a1bf44e13cf2d44d2ea2e928a8140d5d667304abe1a61c4d55b4906f389fbe64 \ ++ --hash=sha256:aa30cd16ddd2f216d07ba01d9635c873e97ddb041c61cf0847254edc37d1c60e \ ++ --hash=sha256:acda193f440dd88c2023cb00aa8bd7b93a9df59978306d14d87a8b12fe426b05 \ ++ --hash=sha256:bd4911c40a43a821dfd93038ac824b756b6e703e26e951718522d29f6eb166a8 \ ++ --hash=sha256:be1099a8295b1a722d03fb7b48be895d30f4301419a583dcf50e9045968a041c \ ++ --hash=sha256:c126fb72be2518395cc0465d4bae03125119136462e1945aea19840e45d89cfc \ ++ --hash=sha256:c53338613043038005bf2e41a2fafa08d29cdbc0ce80891b5366c819456c1ae9 \ ++ --hash=sha256:c789236366525c3ee3cd6e4e450a9ff629a7d1f4d88b8e18a0aea0615fd7ecf8 \ ++ --hash=sha256:cf0ec79e8ca7077f455d14d915d629385153b6a11abc0b93283ed73a8013e376 \ ++ --hash=sha256:d15f060bc6d0964a8bb70aba8f0cb6d11ae99715438f640cff11bbcf172eb0e8 \ ++ --hash=sha256:d284bf68daffc57516535f752e290609b3b643f4bd54b28fc13cb16a89a8bda6 \ ++ --hash=sha256:dabbf3c14de75a20cc3c30bf0c6527157224a93dfb605838eabb1a2ee3be008d \ ++ --hash=sha256:dbbc5b254c36c37d10abb50e899bc3939bbb7ab1e7c659614409af99bd3e7675 \ ++ --hash=sha256:dfc320f08ea9a7ec5b2403dc4e8150636f0d6150f4b9792faaae539c88e7db3b \ ++ --hash=sha256:e2d509786344aa844ae243f68f833ca1ac92ac3e35a92ae038e2ceb44aa355ef \ ++ --hash=sha256:e37469602473f41221cea93fd3736708f561f0fa08ab6b2873dd962014390d52 \ ++ --hash=sha256:ed162b2227f98d5b270ecbe1d53be56c8c81db08a1a8f5f02d89c7bb4d19591d \ ++ --hash=sha256:efe020c46ce3c3a89af6baec6569635812129df6fb6cf76d4943af3ba6ee2069 \ ++ --hash=sha256:f1c5f1f818b669875d191323a48912d3fcd2e4906410e8297bb09ac50c4d5ccc \ ++ --hash=sha256:f25001a955073b80510c0c3db0e043dbbc36904fd69e511c74e3d8640b8a5111 \ ++ --hash=sha256:f3867dc225d9423c245a51eaac2cfcd53dde8e0a8d8090bb6aed6e31bd6c2d4f \ ++ --hash=sha256:f513b2c6c0d5c491f478422f6b5b5c27ac1af06a54c93ef8631806f7231bd92e \ ++ --hash=sha256:f6e42c1bc985d9beee884780ae6048790eb4cd565c46251932906bdb1630034a ++ # via apache-beam ++pymysql==1.1.2 \ ++ --hash=sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03 \ ++ --hash=sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9 ++ # via apache-beam ++pyparsing==3.3.2 \ ++ --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ ++ --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc ++ # via httplib2 ++python-dateutil==2.9.0.post0 \ ++ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ ++ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 ++ # via ++ # apache-beam ++ # betterproto ++ # google-cloud-bigquery ++ # pg8000 ++python-tds==1.17.1 \ ++ --hash=sha256:35cb210b1a54e5ccc91570a83d4e9a2a16682cbeb00bede06fd6cdf9afa9762f \ ++ --hash=sha256:c97483a9adf1dcab8bee66e83429acc502753f389d134553edd818348b94ced0 ++ # via apache-beam ++pytz==2026.1.post1 \ ++ --hash=sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1 \ ++ --hash=sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a ++ # via apache-beam ++pyyaml==6.0.3 \ ++ --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ ++ --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ ++ --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ ++ --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ ++ --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ ++ --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ ++ --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ ++ --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ ++ --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ ++ --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ ++ --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ ++ --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ ++ --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ ++ --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ ++ --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ ++ --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ ++ --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ ++ --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ ++ --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ ++ --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ ++ --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ ++ --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ ++ --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ ++ --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ ++ --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ ++ --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ ++ --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ ++ --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ ++ --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ ++ --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ ++ --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ ++ --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ ++ --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ ++ --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ ++ --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ ++ --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ ++ --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ ++ --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ ++ --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ ++ --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ ++ --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ ++ --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ ++ --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ ++ --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ ++ --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ ++ --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ ++ --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ ++ --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ ++ --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ ++ --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ ++ --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ ++ --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ ++ --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ ++ --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ ++ --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ ++ --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ ++ --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ ++ --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ ++ --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ ++ --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ ++ --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ ++ --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ ++ --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ ++ --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ ++ --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ ++ --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ ++ --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ ++ --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ ++ --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ ++ --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ ++ --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ ++ --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ ++ --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 ++ # via apache-beam ++regex==2026.2.28 \ ++ --hash=sha256:00945d007fd74a9084d2ab79b695b595c6b7ba3698972fadd43e23230c6979c1 \ ++ --hash=sha256:00f2b8d9615aa165fdff0a13f1a92049bfad555ee91e20d246a51aa0b556c60a \ ++ --hash=sha256:01d65fd24206c8e1e97e2e31b286c59009636c022eb5d003f52760b0f42155d4 \ ++ --hash=sha256:02473c954af35dd2defeb07e44182f5705b30ea3f351a7cbffa9177beb14da5d \ ++ --hash=sha256:03a83cc26aa2acda6b8b9dfe748cf9e84cbd390c424a1de34fdcef58961a297a \ ++ --hash=sha256:09500be324f49b470d907b3ef8af9afe857f5cca486f853853f7945ddbf75911 \ ++ --hash=sha256:0b1d2b07614d95fa2bf8a63fd1e98bd8fa2b4848dc91b1efbc8ba219fdd73952 \ ++ --hash=sha256:0d25a10811de831c2baa6aef3c0be91622f44dd8d31dd12e69f6398efb15e48b \ ++ --hash=sha256:0d5bef2031cbf38757a0b0bc4298bb4824b6332d28edc16b39247228fbdbad97 \ ++ --hash=sha256:10d28e19bd4888e4abf43bd3925f3c134c52fdf7259219003588a42e24c2aa25 \ ++ --hash=sha256:180e08a435a0319e6a4821c3468da18dc7001987e1c17ae1335488dfe7518dd8 \ ++ --hash=sha256:195237dc327858a7721bf8b0bbbef797554bc13563c3591e91cd0767bacbe359 \ ++ --hash=sha256:19a9c9e0a8f24f39d575a6a854d516b48ffe4cbdcb9de55cb0570a032556ecff \ ++ --hash=sha256:1c2c95e1a2b0f89d01e821ff4de1be4b5d73d1f4b0bf679fa27c1ad8d2327f1a \ ++ --hash=sha256:1d367257cd86c1cbb97ea94e77b373a0bbc2224976e247f173d19e8f18b4afa7 \ ++ --hash=sha256:1e496956106fd59ba6322a8ea17141a27c5040e5ee8f9433ae92d4e5204462a0 \ ++ --hash=sha256:1f8b17be5c27a684ea6759983c13506bd77bfc7c0347dff41b18ce5ddd2ee09a \ ++ --hash=sha256:2234059cfe33d9813a3677ef7667999caea9eeaa83fef98eb6ce15c6cf9e0215 \ ++ --hash=sha256:25b6eb660c5cf4b8c3407a1ed462abba26a926cc9965e164268a3267bcc06a43 \ ++ --hash=sha256:2954379dd20752e82d22accf3ff465311cbb2bac6c1f92c4afd400e1757f7451 \ ++ --hash=sha256:2afa673660928d0b63d84353c6c08a8a476ddfc4a47e11742949d182e6863ce8 \ ++ --hash=sha256:2b2b23587b26496ff5fd40df4278becdf386813ec00dc3533fa43a4cf0e2ad3c \ ++ --hash=sha256:2fb950ac1d88e6b6a9414381f403797b236f9fa17e1eee07683af72b1634207b \ ++ --hash=sha256:3935174fa4d9f70525a4367aaff3cb8bc0548129d114260c29d9dfa4a5b41692 \ ++ --hash=sha256:39bb5727650b9a0275c6a6690f9bb3fe693a7e6cc5c3155b1240aedf8926423e \ ++ --hash=sha256:3b24bd7e9d85dc7c6a8bd2aa14ecd234274a0248335a02adeb25448aecdd420d \ ++ --hash=sha256:4390c365fd2d45278f45afd4673cb90f7285f5701607e3ad4274df08e36140ae \ ++ --hash=sha256:481df4623fa4969c8b11f3433ed7d5e3dc9cec0f008356c3212b3933fb77e3d8 \ ++ --hash=sha256:4f5c0b182ad4269e7381b7c27fdb0408399881f7a92a4624fd5487f2971dfc11 \ ++ --hash=sha256:50c2fc924749543e0eacc93ada6aeeb3ea5f6715825624baa0dccaec771668ae \ ++ --hash=sha256:511f7419f7afab475fd4d639d4aedfc54205bcb0800066753ef68a59f0f330b5 \ ++ --hash=sha256:516604edd17b1c2c3e579cf4e9b25a53bf8fa6e7cedddf1127804d3e0140ca64 \ ++ --hash=sha256:52b017b35ac2214d0db5f4f90e303634dc44e4aba4bd6235a27f97ecbe5b0472 \ ++ --hash=sha256:5a932ea8ad5d0430351ff9c76c8db34db0d9f53c1d78f06022a21f4e290c5c18 \ ++ --hash=sha256:5cdcc17d935c8f9d3f4db5c2ebe2640c332e3822ad5d23c2f8e0228e6947943a \ ++ --hash=sha256:5d10303dd18cedfd4d095543998404df656088240bcfd3cd20a8f95b861f74bd \ ++ --hash=sha256:5e68192bb3a1d6fb2836da24aa494e413ea65853a21505e142e5b1064a595f3d \ ++ --hash=sha256:64e7c6ad614573e0640f271e811a408d79a9e1fe62a46adb602f598df42a818d \ ++ --hash=sha256:6591f281cb44dc13de9585b552cec6fc6cf47fb2fe7a48892295ee9bc4a612f9 \ ++ --hash=sha256:69fc560ccbf08a09dc9b52ab69cacfae51e0ed80dc5693078bdc97db2f91ae96 \ ++ --hash=sha256:6d63a07e5ec8ce7184452cb00c41c37b49e67dc4f73b2955b5b8e782ea970784 \ ++ --hash=sha256:6db7bfae0f8a2793ff1f7021468ea55e2699d0790eb58ee6ab36ae43aa00bc5b \ ++ --hash=sha256:71a911098be38c859ceb3f9a9ce43f4ed9f4c6720ad8684a066ea246b76ad9ff \ ++ --hash=sha256:73cdcdbba8028167ea81490c7f45280113e41db2c7afb65a276f4711fa3bcbff \ ++ --hash=sha256:78454178c7df31372ea737996fb7f36b3c2c92cccc641d251e072478afb4babc \ ++ --hash=sha256:7900157786428a79615a8264dac1f12c9b02957c473c8110c6b1f972dcecaddf \ ++ --hash=sha256:7ab218076eb0944549e7fe74cf0e2b83a82edb27e81cc87411f76240865e04d5 \ ++ --hash=sha256:7c1b34dfa72f826f535b20712afa9bb3ba580020e834f3c69866c5bddbf10098 \ ++ --hash=sha256:851fa70df44325e1e4cdb79c5e676e91a78147b1b543db2aec8734d2add30ec2 \ ++ --hash=sha256:864cdd1a2ef5716b0ab468af40139e62ede1b3a53386b375ec0786bb6783fc05 \ ++ --hash=sha256:8710d61737b0c0ce6836b1da7109f20d495e49b3809f30e27e9560be67a257bf \ ++ --hash=sha256:9036b400b20e4858d56d117108d7813ed07bb7803e3eed766675862131135ca6 \ ++ --hash=sha256:9185cc63359862a6e80fe97f696e04b0ad9a11c4ac0a4a927f979f611bfe3768 \ ++ --hash=sha256:948c12ef30ecedb128903c2c2678b339746eb7c689c5c21957c4a23950c96d15 \ ++ --hash=sha256:94d63db12e45a9b9f064bfe4800cefefc7e5f182052e4c1b774d46a40ab1d9bb \ ++ --hash=sha256:96f6269a2882fbb0ee76967116b83679dc628e68eaea44e90884b8d53d833881 \ ++ --hash=sha256:97054c55db06ab020342cc0d35d6f62a465fa7662871190175f1ad6c655c028f \ ++ --hash=sha256:98adf340100cbe6fbaf8e6dc75e28f2c191b1be50ffefe292fb0e6f6eefdb0d8 \ ++ --hash=sha256:99985a2c277dcb9ccb63f937451af5d65177af1efdeb8173ac55b61095a0a05c \ ++ --hash=sha256:9b65d33a17101569f86d9c5966a8b1d7fbf8afdda5a8aa219301b0a80f58cf7d \ ++ --hash=sha256:9dd450db6458387167e033cfa80887a34c99c81d26da1bf8b0b41bf8c9cac88e \ ++ --hash=sha256:a25c7701e4f7a70021db9aaf4a4a0a67033c6318752146e03d1b94d32006217e \ ++ --hash=sha256:a448af01e3d8031c89c5d902040b124a5e921a25c4e5e07a861ca591ce429341 \ ++ --hash=sha256:a5dac14d0872eeb35260a8e30bac07ddf22adc1e3a0635b52b02e180d17c9c7e \ ++ --hash=sha256:a729e47d418ea11d03469f321aaf67cdee8954cde3ff2cf8403ab87951ad10f2 \ ++ --hash=sha256:aaffaecffcd2479ce87aa1e74076c221700b7c804e48e98e62500ee748f0f550 \ ++ --hash=sha256:b059e71ec363968671693a78c5053bd9cb2fe410f9b8e4657e88377ebd603a2e \ ++ --hash=sha256:b387a0d092dac157fb026d737dde35ff3e49ef27f285343e7c6401851239df27 \ ++ --hash=sha256:b389c61aa28a79c2e0527ac36da579869c2e235a5b208a12c5b5318cda2501d8 \ ++ --hash=sha256:b42f7466e32bf15a961cf09f35fa6323cc72e64d3d2c990b10de1274a5da0a59 \ ++ --hash=sha256:b49eb78048c6354f49e91e4b77da21257fecb92256b6d599ae44403cab30b05b \ ++ --hash=sha256:b5acd4b6a95f37c3c3828e5d053a7d4edaedb85de551db0153754924cb7c83e3 \ ++ --hash=sha256:b8b3f1be1738feadc69f62daa250c933e85c6f34fa378f54a7ff43807c1b9117 \ ++ --hash=sha256:b8cf76f1a29f0e99dcfd7aef1551a9827588aae5a737fe31442021165f1920dc \ ++ --hash=sha256:ba55c50f408fb5c346a3a02d2ce0ebc839784e24f7c9684fde328ff063c3cdea \ ++ --hash=sha256:bba2b18d70eeb7b79950f12f633beeecd923f7c9ad6f6bae28e59b4cb3ab046b \ ++ --hash=sha256:bbb882061f742eb5d46f2f1bd5304055be0a66b783576de3d7eef1bed4778a6e \ ++ --hash=sha256:bcb399ed84eabf4282587ba151f2732ad8168e66f1d3f85b1d038868fe547703 \ ++ --hash=sha256:bd477d5f79920338107f04aa645f094032d9e3030cc55be581df3d1ef61aa318 \ ++ --hash=sha256:bec23c11cbbf09a4df32fe50d57cbdd777bc442269b6e39a1775654f1c95dee2 \ ++ --hash=sha256:c0b5ccbb8ffb433939d248707d4a8b31993cb76ab1a0187ca886bf50e96df952 \ ++ --hash=sha256:c15af43c72a7fb0c97cbc66fa36a43546eddc5c06a662b64a0cbf30d6ac40944 \ ++ --hash=sha256:c7815afb0ca45456613fdaf60ea9c993715511c8d53a83bc468305cbc0ee23c7 \ ++ --hash=sha256:cb3b1db8ff6c7b8bf838ab05583ea15230cb2f678e569ab0e3a24d1e8320940b \ ++ --hash=sha256:d0b02e8b7e5874b48ae0f077ecca61c1a6a9f9895e9c6dfb191b55b242862033 \ ++ --hash=sha256:d6b08a06976ff4fb0d83077022fde3eca06c55432bb997d8c0495b9a4e9872f4 \ ++ --hash=sha256:d6cfe798d8da41bb1862ed6e0cba14003d387c3c0c4a5d45591076ae9f0ce2f8 \ ++ --hash=sha256:d8511a01d0e4ee1992eb3ba19e09bc1866fe03f05129c3aec3fdc4cbc77aad3f \ ++ --hash=sha256:dc8ed8c3f41c27acb83f7b6a9eb727a73fc6663441890c5cb3426a5f6a91ce7d \ ++ --hash=sha256:dd8847c4978bc3c7e6c826fb745f5570e518b8459ac2892151ce6627c7bc00d5 \ ++ --hash=sha256:de0cf053139f96219ccfabb4a8dd2d217c8c82cb206c91d9f109f3f552d6b43d \ ++ --hash=sha256:dee50f1be42222f89767b64b283283ef963189da0dda4a515aa54a5563c62dec \ ++ --hash=sha256:e1e7b24cb3ae9953a560c563045d1ba56ee4749fbd05cf21ba571069bd7be81b \ ++ --hash=sha256:e59bc8f30414d283ae8ee1617b13d8112e7135cb92830f0ec3688cb29152585a \ ++ --hash=sha256:e61eea47230eba62a31f3e8a0e3164d0f37ef9f40529fb2c79361bc6b53d2a92 \ ++ --hash=sha256:e621fb7c8dc147419b28e1702f58a0177ff8308a76fa295c71f3e7827849f5d9 \ ++ --hash=sha256:e71dcecaa113eebcc96622c17692672c2d104b1d71ddf7adeda90da7ddeb26fc \ ++ --hash=sha256:e7ce83654d1ab701cb619285a18a8e5a889c1216d746ddc710c914ca5fd71022 \ ++ --hash=sha256:e8c8cb2deba42f5ec1ede46374e990f8adc5e6456a57ac1a261b19be6f28e4e6 \ ++ --hash=sha256:ec0c608b7a7465ffadb344ed7c987ff2f11ee03f6a130b569aa74d8a70e8333c \ ++ --hash=sha256:ec6f5674c5dc836994f50f1186dd1fafde4be0666aae201ae2fcc3d29d8adf27 \ ++ --hash=sha256:edb1b1b3a5576c56f08ac46f108c40333f222ebfd5cf63afdfa3aab0791ebe5b \ ++ --hash=sha256:ef77bdde9c9eba3f7fa5b58084b29bbcc74bcf55fdbeaa67c102a35b5bd7e7cc \ ++ --hash=sha256:f2791948f7c70bb9335a9102df45e93d428f4b8128020d85920223925d73b9e1 \ ++ --hash=sha256:f467cb602f03fbd1ab1908f68b53c649ce393fde056628dc8c7e634dab6bfc07 \ ++ --hash=sha256:f8ed9a5d4612df9d4de15878f0bc6aa7a268afbe5af21a3fdd97fa19516e978c \ ++ --hash=sha256:fa539be029844c0ce1114762d2952ab6cfdd7c7c9bd72e0db26b94c3c36dcc5a \ ++ --hash=sha256:fb1c4ff62277d87a7335f2c1ea4e0387b8f2b3ad88a64efd9943906aafad4f33 \ ++ --hash=sha256:fb4db2f17e6484904f986c5a657cec85574c76b5c5e61c7aae9ffa1bc6224f95 \ ++ --hash=sha256:fb66e5245db9652abd7196ace599b04d9c0e4aa7c8f0e2803938377835780081 \ ++ --hash=sha256:fc48c500838be6882b32748f60a15229d2dea96e59ef341eaa96ec83538f498d \ ++ --hash=sha256:fcf26c3c6d0da98fada8ae4ef0aa1c3405a431c0a77eb17306d38a89b02adcd7 \ ++ --hash=sha256:fd0ce43e71d825b7c0661f9c54d4d74bd97c56c3fd102a8985bcfea48236bacb \ ++ --hash=sha256:fd63453f10d29097cc3dc62d070746523973fb5aa1c66d25f8558bebd47fed61 ++ # via apache-beam ++requests==2.32.5 \ ++ --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ ++ --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf ++ # via ++ # apache-beam ++ # cloud-sql-python-connector ++ # google-api-core ++ # google-auth ++ # google-cloud-bigquery ++ # google-cloud-storage ++ # google-genai ++ # keyrings-google-artifactregistry-auth ++ # opentelemetry-resourcedetector-gcp ++rsa==4.9.1 \ ++ --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ ++ --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 ++ # via ++ # google-auth ++ # oauth2client ++scramp==1.4.8 \ ++ --hash=sha256:87c2f15976845a2872fe5490a06097f0d01813cceb53774ea168c911f2ad025c \ ++ --hash=sha256:bd018fabfe46343cceeb9f1c3e8d23f55770271e777e3accbfaee3ff0a316e71 ++ # via pg8000 ++secretstorage==3.5.0 \ ++ --hash=sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137 \ ++ --hash=sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be ++ # via keyring ++six==1.17.0 \ ++ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ ++ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 ++ # via ++ # google-apitools ++ # oauth2client ++ # python-dateutil ++sniffio==1.3.1 \ ++ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ ++ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc ++ # via google-genai ++sortedcontainers==2.4.0 \ ++ --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \ ++ --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 ++ # via apache-beam ++sqlparse==0.5.5 \ ++ --hash=sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba \ ++ --hash=sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e ++ # via google-cloud-spanner ++tenacity==9.1.4 \ ++ --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 \ ++ --hash=sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a ++ # via google-genai ++typing-extensions==4.15.0 \ ++ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ ++ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 ++ # via ++ # aiosignal ++ # anyio ++ # apache-beam ++ # betterproto ++ # google-cloud-aiplatform ++ # google-genai ++ # opentelemetry-api ++ # opentelemetry-resourcedetector-gcp ++ # opentelemetry-sdk ++ # opentelemetry-semantic-conventions ++ # pydantic ++ # pydantic-core ++ # typing-inspection ++typing-inspection==0.4.2 \ ++ --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ ++ --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 ++ # via pydantic ++urllib3==2.6.3 \ ++ --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ ++ --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 ++ # via requests ++websockets==16.0 \ ++ --hash=sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c \ ++ --hash=sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a \ ++ --hash=sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe \ ++ --hash=sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e \ ++ --hash=sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec \ ++ --hash=sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1 \ ++ --hash=sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64 \ ++ --hash=sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3 \ ++ --hash=sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8 \ ++ --hash=sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206 \ ++ --hash=sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3 \ ++ --hash=sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156 \ ++ --hash=sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d \ ++ --hash=sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9 \ ++ --hash=sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad \ ++ --hash=sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2 \ ++ --hash=sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03 \ ++ --hash=sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8 \ ++ --hash=sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230 \ ++ --hash=sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8 \ ++ --hash=sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea \ ++ --hash=sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641 \ ++ --hash=sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957 \ ++ --hash=sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6 \ ++ --hash=sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6 \ ++ --hash=sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5 \ ++ --hash=sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f \ ++ --hash=sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00 \ ++ --hash=sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e \ ++ --hash=sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b \ ++ --hash=sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72 \ ++ --hash=sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39 \ ++ --hash=sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9 \ ++ --hash=sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79 \ ++ --hash=sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0 \ ++ --hash=sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac \ ++ --hash=sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35 \ ++ --hash=sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0 \ ++ --hash=sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5 \ ++ --hash=sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c \ ++ --hash=sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8 \ ++ --hash=sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1 \ ++ --hash=sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244 \ ++ --hash=sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3 \ ++ --hash=sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767 \ ++ --hash=sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a \ ++ --hash=sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d \ ++ --hash=sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd \ ++ --hash=sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e \ ++ --hash=sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944 \ ++ --hash=sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82 \ ++ --hash=sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d \ ++ --hash=sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4 \ ++ --hash=sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5 \ ++ --hash=sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904 \ ++ --hash=sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde \ ++ --hash=sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f \ ++ --hash=sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c \ ++ --hash=sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89 \ ++ --hash=sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da \ ++ --hash=sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4 ++ # via google-genai ++yarl==1.23.0 \ ++ --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \ ++ --hash=sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4 \ ++ --hash=sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85 \ ++ --hash=sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993 \ ++ --hash=sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222 \ ++ --hash=sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de \ ++ --hash=sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25 \ ++ --hash=sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e \ ++ --hash=sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2 \ ++ --hash=sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e \ ++ --hash=sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860 \ ++ --hash=sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957 \ ++ --hash=sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760 \ ++ --hash=sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52 \ ++ --hash=sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788 \ ++ --hash=sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912 \ ++ --hash=sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719 \ ++ --hash=sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035 \ ++ --hash=sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220 \ ++ --hash=sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412 \ ++ --hash=sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05 \ ++ --hash=sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41 \ ++ --hash=sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 \ ++ --hash=sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4 \ ++ --hash=sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd \ ++ --hash=sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748 \ ++ --hash=sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a \ ++ --hash=sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4 \ ++ --hash=sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34 \ ++ --hash=sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069 \ ++ --hash=sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25 \ ++ --hash=sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2 \ ++ --hash=sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb \ ++ --hash=sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f \ ++ --hash=sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5 \ ++ --hash=sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8 \ ++ --hash=sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c \ ++ --hash=sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512 \ ++ --hash=sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6 \ ++ --hash=sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5 \ ++ --hash=sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9 \ ++ --hash=sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072 \ ++ --hash=sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5 \ ++ --hash=sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277 \ ++ --hash=sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a \ ++ --hash=sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6 \ ++ --hash=sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae \ ++ --hash=sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26 \ ++ --hash=sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2 \ ++ --hash=sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4 \ ++ --hash=sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70 \ ++ --hash=sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723 \ ++ --hash=sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c \ ++ --hash=sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9 \ ++ --hash=sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5 \ ++ --hash=sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e \ ++ --hash=sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c \ ++ --hash=sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4 \ ++ --hash=sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0 \ ++ --hash=sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2 \ ++ --hash=sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b \ ++ --hash=sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7 \ ++ --hash=sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750 \ ++ --hash=sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2 \ ++ --hash=sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474 \ ++ --hash=sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716 \ ++ --hash=sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7 \ ++ --hash=sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123 \ ++ --hash=sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007 \ ++ --hash=sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595 \ ++ --hash=sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe \ ++ --hash=sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea \ ++ --hash=sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598 \ ++ --hash=sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679 \ ++ --hash=sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8 \ ++ --hash=sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83 \ ++ --hash=sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6 \ ++ --hash=sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f \ ++ --hash=sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94 \ ++ --hash=sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51 \ ++ --hash=sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120 \ ++ --hash=sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039 \ ++ --hash=sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1 \ ++ --hash=sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05 \ ++ --hash=sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb \ ++ --hash=sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144 \ ++ --hash=sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa \ ++ --hash=sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a \ ++ --hash=sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99 \ ++ --hash=sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928 \ ++ --hash=sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d \ ++ --hash=sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3 \ ++ --hash=sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434 \ ++ --hash=sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86 \ ++ --hash=sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46 \ ++ --hash=sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319 \ ++ --hash=sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67 \ ++ --hash=sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c \ ++ --hash=sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169 \ ++ --hash=sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c \ ++ --hash=sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59 \ ++ --hash=sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107 \ ++ --hash=sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4 \ ++ --hash=sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a \ ++ --hash=sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb \ ++ --hash=sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f \ ++ --hash=sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769 \ ++ --hash=sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432 \ ++ --hash=sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090 \ ++ --hash=sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764 \ ++ --hash=sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d \ ++ --hash=sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4 \ ++ --hash=sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b \ ++ --hash=sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d \ ++ --hash=sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543 \ ++ --hash=sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24 \ ++ --hash=sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5 \ ++ --hash=sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b \ ++ --hash=sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d \ ++ --hash=sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b \ ++ --hash=sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6 \ ++ --hash=sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735 \ ++ --hash=sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e \ ++ --hash=sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28 \ ++ --hash=sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3 \ ++ --hash=sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401 \ ++ --hash=sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6 \ ++ --hash=sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d ++ # via aiohttp ++zipp==3.23.0 \ ++ --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ ++ --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 ++ # via importlib-metadata ++zstandard==0.25.0 \ ++ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ ++ --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ ++ --hash=sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3 \ ++ --hash=sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f \ ++ --hash=sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6 \ ++ --hash=sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936 \ ++ --hash=sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431 \ ++ --hash=sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250 \ ++ --hash=sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa \ ++ --hash=sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f \ ++ --hash=sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851 \ ++ --hash=sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3 \ ++ --hash=sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9 \ ++ --hash=sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6 \ ++ --hash=sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362 \ ++ --hash=sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649 \ ++ --hash=sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb \ ++ --hash=sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5 \ ++ --hash=sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439 \ ++ --hash=sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137 \ ++ --hash=sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa \ ++ --hash=sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd \ ++ --hash=sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701 \ ++ --hash=sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0 \ ++ --hash=sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043 \ ++ --hash=sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1 \ ++ --hash=sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860 \ ++ --hash=sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611 \ ++ --hash=sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53 \ ++ --hash=sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b \ ++ --hash=sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088 \ ++ --hash=sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e \ ++ --hash=sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa \ ++ --hash=sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2 \ ++ --hash=sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0 \ ++ --hash=sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7 \ ++ --hash=sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf \ ++ --hash=sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388 \ ++ --hash=sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530 \ ++ --hash=sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577 \ ++ --hash=sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902 \ ++ --hash=sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc \ ++ --hash=sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98 \ ++ --hash=sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a \ ++ --hash=sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097 \ ++ --hash=sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea \ ++ --hash=sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09 \ ++ --hash=sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb \ ++ --hash=sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7 \ ++ --hash=sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74 \ ++ --hash=sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b \ ++ --hash=sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b \ ++ --hash=sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b \ ++ --hash=sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91 \ ++ --hash=sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150 \ ++ --hash=sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049 \ ++ --hash=sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27 \ ++ --hash=sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a \ ++ --hash=sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00 \ ++ --hash=sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd \ ++ --hash=sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072 \ ++ --hash=sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c \ ++ --hash=sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c \ ++ --hash=sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065 \ ++ --hash=sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512 \ ++ --hash=sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1 \ ++ --hash=sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f \ ++ --hash=sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2 \ ++ --hash=sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df \ ++ --hash=sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab \ ++ --hash=sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7 \ ++ --hash=sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b \ ++ --hash=sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550 \ ++ --hash=sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0 \ ++ --hash=sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea \ ++ --hash=sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277 \ ++ --hash=sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2 \ ++ --hash=sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7 \ ++ --hash=sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778 \ ++ --hash=sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859 \ ++ --hash=sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d \ ++ --hash=sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751 \ ++ --hash=sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12 \ ++ --hash=sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2 \ ++ --hash=sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d \ ++ --hash=sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0 \ ++ --hash=sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3 \ ++ --hash=sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd \ ++ --hash=sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e \ ++ --hash=sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f \ ++ --hash=sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e \ ++ --hash=sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94 \ ++ --hash=sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708 \ ++ --hash=sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313 \ ++ --hash=sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4 \ ++ --hash=sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c \ ++ --hash=sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344 \ ++ --hash=sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551 \ ++ --hash=sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01 ++ # via apache-beam ++ ++# The following packages are considered to be unsafe in a requirements file: ++setuptools==82.0.0 \ ++ --hash=sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb \ ++ --hash=sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0 ++ # via -r python/default_base_bqmonitor_requirements.txt ++ +diff --git a/python/src/main/python/bigquery-anomaly-detection/setup.py b/python/src/main/python/bigquery-anomaly-detection/setup.py +new file mode 100644 +index 000000000..067561292 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/setup.py +@@ -0,0 +1,14 @@ ++from setuptools import setup, find_packages ++ ++setup( ++ name='bqmonitor', ++ version='0.1.0', ++ description='BigQuery anomaly monitoring pipeline (Dataflow Flex Template)', ++ package_dir={'': 'src'}, ++ packages=find_packages(where='src'), ++ python_requires='>=3.11', ++ install_requires=[ ++ 'apache-beam[gcp]==2.71.0', ++ 'google-cloud-bigquery-storage', ++ ], ++) +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/__init__.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/__init__.py +new file mode 100644 +index 000000000..e69de29bb +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py +new file mode 100644 +index 000000000..0bb98127a +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py +@@ -0,0 +1,1402 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Streaming source for BigQuery change history (APPENDS/CHANGES functions). ++ ++This module provides ``ReadBigQueryChangeHistory``, a streaming PTransform ++that continuously polls BigQuery APPENDS() or CHANGES() functions and emits ++changed rows as an unbounded PCollection. ++ ++**Status: Experimental**: API may change without notice. ++ ++Usage:: ++ ++ import apache_beam as beam ++ from bqmonitor.cdc import ReadBigQueryChangeHistory ++ ++ with beam.Pipeline(options=pipeline_options) as p: ++ changes = ( ++ p ++ | ReadBigQueryChangeHistory( ++ table='my-project:my_dataset.my_table', ++ change_function='APPENDS', ++ poll_interval_sec=60)) ++ ++Architecture: ++ Poll: Polling SDF emits lightweight _QueryRange instructions. ++ Query: _ExecuteQueryFn runs the BQ query, writes to a temp table. ++ Read: SDF reads temp table via Storage Read API with dynamic splitting. ++ Cleanup: Stateful DoFn tracks stream completion, deletes temp tables. ++""" ++ ++import dataclasses ++import datetime ++import logging ++import random ++import sys ++import time ++import uuid ++from typing import Any ++from typing import Dict ++from typing import Iterable ++from typing import List ++from typing import Optional ++from typing import Tuple ++ ++import apache_beam as beam ++from apache_beam.io.gcp import bigquery_tools ++from apache_beam.io.gcp.internal.clients import bigquery ++from apache_beam.io.iobase import WatermarkEstimator ++from apache_beam.io.restriction_trackers import OffsetRange ++from apache_beam.io.restriction_trackers import OffsetRestrictionTracker ++from apache_beam.io.watermark_estimators import ManualWatermarkEstimator ++from apache_beam.metrics import Metrics ++from apache_beam.transforms.core import WatermarkEstimatorProvider ++from apache_beam.transforms import trigger as beam_trigger ++from apache_beam.transforms.window import GlobalWindows ++from apache_beam.transforms.window import TimestampedValue ++from apache_beam.utils import retry ++from apache_beam.utils.timestamp import MAX_TIMESTAMP ++from apache_beam.utils.timestamp import Duration ++from apache_beam.utils.timestamp import Timestamp ++ ++try: ++ from apitools.base.py.exceptions import HttpError ++except ImportError: ++ HttpError = None # type: ignore ++ ++try: ++ from google.cloud import bigquery_storage_v1 as bq_storage ++except ImportError: ++ bq_storage = None # type: ignore ++ ++try: ++ import pyarrow ++except ImportError: ++ pyarrow = None # type: ignore ++ ++_LOGGER = logging.getLogger(__name__) ++ ++__all__ = ['ReadBigQueryChangeHistory'] ++ ++# Max time range for CHANGES() queries: 1 day. ++_MAX_CHANGES_RANGE = Duration(seconds=86400) ++ ++# Side output tag for cleanup signals between the Read SDF and Cleanup DoFn. ++_CLEANUP_TAG = 'cleanup' ++ ++# Default number of Storage Read API streams to request. ++# Matches ReadFromBigQuery's MIN_SPLIT_COUNT to enable parallelism. ++# The server may return fewer streams if the table is small. ++_DEFAULT_MAX_STREAMS = 10 ++ ++# Default table expiration for auto-created temp datasets: 24 hours in ms. ++# Tables created in the dataset auto-expire after this duration if not ++# explicitly deleted, acting as a safety net for orphaned temp tables ++# (e.g. pipeline crash before cleanup runs). ++_DEFAULT_TABLE_EXPIRATION_MS = 24 * 60 * 60 * 1000 ++ ++ ++@dataclasses.dataclass ++class _QueryResult: ++ """Bridges the Query step (query execution) to the Read SDF. ++ ++ After _ExecuteQueryFn runs a CHANGES/APPENDS query, it emits a _QueryResult ++ pointing to the temp table containing query results. The Read SDF reads ++ rows from that temp table via the Storage Read API. ++ ++ range_start/range_end define the time window this query covers as Beam ++ Timestamps (int microseconds internally). The Read SDF uses range_start ++ to set an initial watermark hold so the runner doesn't advance the ++ watermark past the data's timestamps. ++ """ ++ temp_table_ref: 'bigquery.TableReference' ++ range_start: Timestamp ++ range_end: Timestamp ++ ++ ++@dataclasses.dataclass ++class _PollConfig: ++ """Input element for the polling SDF. ++ ++ Only contains start_time (Beam Timestamp), which ++ _PollWatermarkEstimatorProvider uses to initialize the watermark hold. ++ All other config is passed via _PollChangeHistoryFn.__init__. ++ """ ++ start_time: Timestamp ++ ++ ++@dataclasses.dataclass ++class _QueryRange: ++ """Lightweight instruction emitted by the polling SDF. ++ ++ Contains only the time range to query as Beam Timestamps (int microseconds ++ internally). Static config (table, project, etc.) is held by ++ _ExecuteQueryFn which receives these after a Reshuffle commit boundary, ++ preventing duplicate queries on SDF re-dispatch. ++ """ ++ chunk_start: Timestamp ++ chunk_end: Timestamp ++ ++ ++class _StreamRestriction: ++ """Restriction carrying BQ Storage stream names for cross-worker safety. ++ ++ Unlike a plain OffsetRange(0, N), this restriction is self-contained: ++ each split carries the actual stream name strings so it can be processed ++ on any worker. Composes an OffsetRange for offset logic. ++ """ ++ __slots__ = ('stream_names', 'range') ++ ++ def __init__( ++ self, stream_names: Tuple[str, ...], start: int, stop: int) -> None: ++ self.stream_names = stream_names # tuple of BQ stream name strings ++ self.range = OffsetRange(start, stop) ++ ++ @property ++ def start(self) -> int: ++ return self.range.start ++ ++ @property ++ def stop(self) -> int: ++ return self.range.stop ++ ++ def __eq__(self, other: object) -> bool: ++ if not isinstance(other, _StreamRestriction): ++ return False ++ return ( ++ self.stream_names == other.stream_names and self.range == other.range) ++ ++ def __hash__(self) -> int: ++ return hash((type(self), self.stream_names, self.range)) ++ ++ def __repr__(self) -> str: ++ return ( ++ '_StreamRestriction(streams=%d, start=%d, stop=%d)' % ++ (len(self.stream_names), self.start, self.stop)) ++ ++ def size(self) -> int: ++ return self.range.size() ++ ++ ++class _StreamRestrictionTracker(beam.io.iobase.RestrictionTracker): ++ """Tracker for _StreamRestriction, delegating offset logic to ++ OffsetRestrictionTracker.""" ++ def __init__(self, restriction: _StreamRestriction) -> None: ++ self._stream_names = restriction.stream_names ++ self._offset_tracker = OffsetRestrictionTracker(restriction.range) ++ ++ def current_restriction(self) -> _StreamRestriction: ++ r = self._offset_tracker.current_restriction() ++ return _StreamRestriction(self._stream_names, r.start, r.stop) ++ ++ def try_claim(self, position: int) -> bool: ++ return self._offset_tracker.try_claim(position) ++ ++ def try_split( ++ self, fraction_of_remainder: float ++ ) -> Optional[Tuple[_StreamRestriction, _StreamRestriction]]: ++ result = self._offset_tracker.try_split(fraction_of_remainder) ++ if result is not None: ++ primary, residual = result ++ return ( ++ _StreamRestriction(self._stream_names, primary.start, primary.stop), ++ _StreamRestriction(self._stream_names, residual.start, residual.stop)) ++ return None ++ ++ def check_done(self) -> None: ++ self._offset_tracker.check_done() ++ ++ def current_progress(self): ++ return self._offset_tracker.current_progress() ++ ++ def is_bounded(self) -> bool: ++ return True ++ ++ ++class _NonSplittableOffsetTracker(OffsetRestrictionTracker): ++ """OffsetRestrictionTracker that allows checkpointing but prevents splitting. ++ ++ Checkpointing (fraction=0) is required for defer_remainder(). All other ++ split fractions are refused, ensuring the polling SDF runs as a singleton. ++ """ ++ def try_split( ++ self, fraction_of_remainder: float ++ ) -> Optional[Tuple[OffsetRange, OffsetRange]]: ++ if fraction_of_remainder == 0: ++ return super().try_split(fraction_of_remainder) ++ return None ++ ++ ++class _PollWatermarkEstimator(WatermarkEstimator): ++ """Watermark estimator that tracks both a watermark hold and poll cursor. ++ ++ The watermark hold (reported via current_watermark) is set to start_ts: ++ the earliest data timestamp emitted by the current poll. This prevents ++ downstream stages from seeing data as late. ++ ++ The poll cursor (last_end) tracks where the next poll should start. ++ This is separate from the watermark so we can hold the watermark back ++ at start_ts while still advancing the poll cursor to end_ts. ++ ++ All timestamps are Beam Timestamps (int microseconds internally). ++ ++ State is checkpointed as (watermark_hold, last_end) so ++ both values survive SDF re-dispatch. ++ """ ++ def __init__(self, state: Tuple[Timestamp, Timestamp]) -> None: ++ self._watermark_hold, self._last_end = state ++ ++ def observe_timestamp(self, timestamp: Timestamp) -> None: ++ pass ++ ++ def current_watermark(self) -> Timestamp: ++ return self._watermark_hold ++ ++ def get_estimator_state(self) -> Tuple[Timestamp, Timestamp]: ++ return (self._watermark_hold, self._last_end) ++ ++ def set_watermark(self, timestamp: Timestamp) -> None: ++ if not isinstance(timestamp, Timestamp): ++ raise ValueError('set_watermark expects a Timestamp as input') ++ if self._watermark_hold and self._watermark_hold > timestamp: ++ raise ValueError( ++ 'Watermark must be monotonically increasing. ' ++ 'Provided %s < current %s' % (timestamp, self._watermark_hold)) ++ self._watermark_hold = timestamp ++ ++ def advance_poll_cursor(self, end: Timestamp) -> None: ++ """Record end so the next poll starts from here. ++ ++ Only advances forward: if end is earlier than the current cursor ++ (e.g. BQ clock regression), the cursor stays put so the next poll ++ doesn't re-query an already-covered range. ++ """ ++ self._last_end = max(self._last_end, end) ++ ++ def poll_cursor(self) -> Timestamp: ++ """Return the start Timestamp for the next poll.""" ++ return self._last_end ++ ++ ++class _PollWatermarkEstimatorProvider(WatermarkEstimatorProvider): ++ """Provider for _PollWatermarkEstimator. ++ ++ Initializes with watermark hold at start_time and poll cursor at ++ start_time (first poll will query from start_time). ++ """ ++ def initial_estimator_state( ++ self, element: _PollConfig, ++ restriction: OffsetRange) -> Tuple[Timestamp, Timestamp]: ++ return (element.start_time, element.start_time) ++ ++ def create_watermark_estimator( ++ self, estimator_state: Tuple[Timestamp, ++ Timestamp]) -> _PollWatermarkEstimator: ++ return _PollWatermarkEstimator(estimator_state) ++ ++ ++def _table_key(table_ref: 'bigquery.TableReference') -> str: ++ """Convert a TableReference to a 'project.dataset.table' string.""" ++ return f'{table_ref.projectId}.{table_ref.datasetId}.{table_ref.tableId}' ++ ++ ++def build_changes_query( ++ table: str, ++ start: Timestamp, ++ end: Timestamp, ++ change_function: str, ++ change_type_column: str = 'change_type', ++ change_timestamp_column: str = 'change_timestamp', ++ columns: Optional[List[str]] = None, ++ row_filter: Optional[str] = None) -> str: ++ """Build a CHANGES() or APPENDS() SQL query. ++ ++ Args: ++ table: Table name as 'project.dataset.table' or 'project:dataset.table'. ++ start: Start timestamp (Beam Timestamp). Inclusive. ++ end: End timestamp (Beam Timestamp). Exclusive. ++ change_function: 'CHANGES' or 'APPENDS'. ++ change_type_column: Output column name for _CHANGE_TYPE pseudo-column. ++ change_timestamp_column: Output column name for _CHANGE_TIMESTAMP ++ pseudo-column. ++ columns: Optional list of column names to select. If None, selects all ++ columns. Pseudo-columns are always appended regardless. ++ row_filter: Optional SQL WHERE clause (without the WHERE keyword). ++ Applied after the CHANGES/APPENDS function. ++ ++ Returns: ++ SQL string. ++ """ ++ # Normalize 'project:dataset.table' to 'project.dataset.table' ++ table = table.replace(':', '.') ++ start_iso = start.to_rfc3339() ++ end_iso = end.to_rfc3339() ++ # Pseudo-columns (_CHANGE_TYPE, _CHANGE_TIMESTAMP) can't be written to ++ # destination tables with their original names. Rename them so they can ++ # be persisted to the temp table for Storage Read API reading. ++ pseudo = ( ++ f"_CHANGE_TYPE AS {change_type_column}, " ++ f"_CHANGE_TIMESTAMP AS {change_timestamp_column}") ++ if columns is None: ++ select = f"SELECT * EXCEPT(_CHANGE_TYPE, _CHANGE_TIMESTAMP), {pseudo}" ++ else: ++ select = f"SELECT {', '.join(columns)}, {pseudo}" ++ from_clause = ( ++ f"FROM {change_function}" ++ f"(TABLE `{table}`, " ++ f"TIMESTAMP '{start_iso}', " ++ f"TIMESTAMP '{end_iso}')") ++ where = f" WHERE {row_filter}" if row_filter else "" ++ return f"{select} {from_clause}{where}" ++ ++ ++def compute_ranges(start: Timestamp, end: Timestamp, ++ change_function: str) -> List[Tuple[Timestamp, Timestamp]]: ++ """Split [start, end) into query-safe chunks. ++ ++ CHANGES() has a max 1-day range. APPENDS() has no limit. ++ ++ Args: ++ start: Start Timestamp. Inclusive. ++ end: End Timestamp. Exclusive. ++ change_function: 'CHANGES' or 'APPENDS'. ++ ++ Returns: ++ List of (start, end) Timestamp tuples. Empty if end <= start. ++ """ ++ if end <= start: ++ return [] ++ ++ if change_function != 'CHANGES': ++ return [(start, end)] ++ ++ # CHANGES: chunk into <=1-day ranges ++ ranges = [] ++ current = start ++ while current < end: ++ chunk_end = min(current + _MAX_CHANGES_RANGE, end) ++ ranges.append((current, chunk_end)) ++ current = chunk_end ++ return ranges ++ ++ ++def _utc(ts: Timestamp) -> str: ++ """Format a Beam Timestamp as a concise UTC string for logging.""" ++ return ts.to_utc_datetime(has_tz=True).strftime('%Y-%m-%dT%H:%M:%S.%f') ++ ++ ++# ============================================================================= ++# Poll: _PollChangeHistoryFn (Polling SDF) ++# ============================================================================= ++ ++ ++class _PollChangeHistoryFn(beam.DoFn, beam.transforms.core.RestrictionProvider): ++ """SDF that periodically emits _QueryRange instructions. ++ ++ Uses defer_remainder() for poll timing and _PollWatermarkEstimator to ++ control the watermark. The watermark is initially held at start_time, then ++ advanced to start_ts of each poll. ++ ++ All timestamps are Beam Timestamps (int microseconds internally). ++ Durations (buffer, poll_interval) are Beam Durations. ++ ++ Derives start_ts from the poll cursor. On each poll: ++ 1. start_ts = poll cursor (last end_ts, or start_time on first poll) ++ 2. end_ts = bq_now - buffer ++ 3. Computes query chunks, yields _QueryRange per chunk ++ 4. Advances poll cursor to end_ts (for next poll's start) ++ 5. Advances watermark to start_ts (earliest data in this poll) ++ 6. Defers to next poll interval ++ """ ++ def __init__( ++ self, ++ table: str, ++ project: str, ++ change_function: str, ++ buffer: Duration, ++ start_time: Timestamp, ++ stop_time: Timestamp, ++ poll_interval: Duration, ++ location: Optional[str] = None) -> None: ++ self._table = table ++ self._project = project ++ self._change_function = change_function ++ self._buffer = buffer ++ self._start_time = start_time ++ self._stop_time = stop_time ++ self._poll_interval = poll_interval ++ self._location = location ++ ++ def setup(self) -> None: ++ self._bq_wrapper = bigquery_tools.BigQueryWrapper() ++ if self._location is None: ++ table_ref = bigquery_tools.parse_table_reference( ++ self._table, project=self._project) ++ self._location = self._bq_wrapper.get_table_location( ++ table_ref.projectId, table_ref.datasetId, table_ref.tableId) ++ _LOGGER.info( ++ '[Poll] Inferred location=%s from source table %s', ++ self._location, ++ self._table) ++ ++ @retry.with_exponential_backoff( ++ num_retries=3, ++ retry_filter=retry.retry_on_server_errors_and_timeout_filter) ++ def _get_bq_timestamp(self) -> Timestamp: ++ """Query BigQuery for the current server timestamp. ++ ++ Returns a Beam Timestamp created from integer microseconds. ++ Uses BQ's CURRENT_TIMESTAMP instead of the local clock to avoid ++ data loss from clock skew between the worker VM and BigQuery. ++ """ ++ request = bigquery.BigqueryJobsQueryRequest( ++ projectId=self._project, ++ queryRequest=bigquery.QueryRequest( ++ query='SELECT UNIX_MICROS(CURRENT_TIMESTAMP()) AS ts', ++ useLegacySql=False, ++ location=self._location)) ++ response = self._bq_wrapper.client.jobs.Query(request) ++ return Timestamp(micros=int(response.rows[0].f[0].v.string_value)) ++ ++ def initial_restriction(self, element: _PollConfig) -> OffsetRange: ++ return OffsetRange(0, sys.maxsize) ++ ++ def create_tracker( ++ self, restriction: OffsetRange) -> _NonSplittableOffsetTracker: ++ # Guarantee at least one poll cycle: restriction.start == 0 on the first ++ # invocation (from initial_restriction). After the first try_claim(0) + ++ # defer_remainder, subsequent invocations arrive with start >= 1. ++ if restriction.start > 0 and time.time() >= float(self._stop_time): ++ _LOGGER.info( ++ '[Poll] create_tracker: stop_time reached, ' ++ 'returning empty range to terminate SDF') ++ return _NonSplittableOffsetTracker( ++ OffsetRange(restriction.start, restriction.start)) ++ return _NonSplittableOffsetTracker(restriction) ++ ++ def restriction_size( ++ self, element: _PollConfig, restriction: OffsetRange) -> int: ++ return 1 ++ ++ def split(self, element: _PollConfig, ++ restriction: OffsetRange) -> Iterable[OffsetRange]: ++ yield restriction ++ ++ def truncate(self, element: _PollConfig, restriction: OffsetRange) -> None: ++ return None ++ ++ def _next_poll_time(self, start_ts: Timestamp, ++ now: float) -> Optional[Timestamp]: ++ """Return a Timestamp to defer to, or None if we should poll now.""" ++ earliest = start_ts + self._buffer + self._poll_interval ++ if now < float(earliest): ++ return earliest ++ return None ++ ++ def _emit_query_ranges( ++ self, ++ start_ts: Timestamp, ++ end_ts: Timestamp, ++ watermark_estimator: _PollWatermarkEstimator) -> Iterable[_QueryRange]: ++ """Compute and yield _QueryRange elements, advancing estimator state.""" ++ ranges = compute_ranges(start_ts, end_ts, self._change_function) ++ _LOGGER.info( ++ '[Poll] %d chunks for [%s, %s)', ++ len(ranges), ++ _utc(start_ts), ++ _utc(end_ts)) ++ Metrics.counter('BigQueryChangeHistory', 'polls').inc() ++ ++ watermark_estimator.advance_poll_cursor(end_ts) ++ watermark_estimator.set_watermark(start_ts) ++ _LOGGER.info( ++ '[Poll] Watermark=%s (start_ts), cursor=%s (end_ts)', ++ _utc(start_ts), ++ _utc(end_ts)) ++ ++ for chunk_start, chunk_end in ranges: ++ yield TimestampedValue( ++ _QueryRange(chunk_start=chunk_start, chunk_end=chunk_end), start_ts) ++ ++ @beam.DoFn.unbounded_per_element() ++ def process( ++ self, ++ _: _PollConfig, ++ restriction_tracker=beam.DoFn.RestrictionParam(), ++ watermark_estimator=beam.DoFn.WatermarkEstimatorParam( ++ _PollWatermarkEstimatorProvider()) ++ ) -> Iterable[_QueryRange]: ++ ++ now = time.time() ++ start_ts = watermark_estimator.poll_cursor() ++ ++ defer_to = self._next_poll_time(start_ts, now) ++ if defer_to is not None: ++ restriction_tracker.defer_remainder(defer_to) ++ return ++ ++ # Use BQ server time instead of local clock to avoid data loss ++ # from clock skew between the worker VM and BigQuery. ++ bq_now = self._get_bq_timestamp() ++ end_ts = min(bq_now - self._buffer, self._stop_time) ++ ++ _LOGGER.info( ++ '[Poll] Polling: start=%s, end=%s, watermark=%s, ' ++ 'clock_skew=%.3fs', ++ _utc(start_ts), ++ _utc(end_ts), ++ _utc(watermark_estimator.current_watermark()), ++ float(bq_now) - now) ++ ++ current_index = restriction_tracker.current_restriction().start ++ ++ if not restriction_tracker.try_claim(current_index): ++ return ++ restriction_tracker.defer_remainder(Timestamp.of(now) + self._poll_interval) ++ ++ yield from self._emit_query_ranges(start_ts, end_ts, watermark_estimator) ++ ++ ++class _ExecuteQueryFn(beam.DoFn): ++ """Executes a BQ CHANGES/APPENDS query from a _QueryRange instruction. ++ """ ++ def __init__( ++ self, ++ table: str, ++ project: str, ++ change_function: str, ++ temp_dataset: str, ++ location: Optional[str], ++ change_type_column: str = 'change_type', ++ change_timestamp_column: str = 'change_timestamp', ++ columns: Optional[List[str]] = None, ++ row_filter: Optional[str] = None) -> None: ++ self._table = table ++ self._project = project ++ self._change_function = change_function ++ self._temp_dataset = temp_dataset ++ self._location = location ++ self._change_type_column = change_type_column ++ self._change_timestamp_column = change_timestamp_column ++ self._columns = columns ++ self._row_filter = row_filter ++ ++ def setup(self) -> None: ++ self._bq_wrapper = bigquery_tools.BigQueryWrapper() ++ if self._location is None: ++ table_ref = bigquery_tools.parse_table_reference( ++ self._table, project=self._project) ++ self._location = self._bq_wrapper.get_table_location( ++ table_ref.projectId, table_ref.datasetId, table_ref.tableId) ++ _LOGGER.info( ++ '[Query] Inferred location=%s from source table %s', ++ self._location, ++ self._table) ++ self._get_or_create_temp_dataset() ++ ++ def _get_or_create_temp_dataset(self) -> None: ++ """Create the temp dataset if it doesn't exist. ++ ++ Sets a default table expiration so orphaned temp tables (e.g. from ++ pipeline crashes before cleanup) are automatically garbage-collected. ++ """ ++ try: ++ self._bq_wrapper.client.datasets.Get( ++ bigquery.BigqueryDatasetsGetRequest( ++ projectId=self._project, datasetId=self._temp_dataset)) ++ except HttpError as e: ++ if e.status_code != 404: ++ raise ++ _LOGGER.info( ++ '[Query] Creating temp dataset %s:%s (location=%s)', ++ self._project, self._temp_dataset, self._location) ++ dataset_ref = bigquery.DatasetReference( ++ projectId=self._project, datasetId=self._temp_dataset) ++ dataset = bigquery.Dataset( ++ datasetReference=dataset_ref, ++ defaultTableExpirationMs=_DEFAULT_TABLE_EXPIRATION_MS) ++ if self._location is not None: ++ dataset.location = self._location ++ self._bq_wrapper.client.datasets.Insert( ++ bigquery.BigqueryDatasetsInsertRequest( ++ projectId=self._project, dataset=dataset)) ++ ++ def process(self, qr: _QueryRange) -> Iterable[_QueryResult]: ++ """Execute the BQ query described by a _QueryRange and yield _QueryResult. ++ """ ++ ++ sql = build_changes_query( ++ self._table, ++ qr.chunk_start, ++ qr.chunk_end, ++ self._change_function, ++ self._change_type_column, ++ self._change_timestamp_column, ++ self._columns, ++ self._row_filter) ++ temp_table_id = f'beam_ch_temp_{uuid.uuid4().hex[:8]}' ++ job_id = f'beam_ch_{uuid.uuid4().hex[:12]}' ++ ++ _LOGGER.info( ++ '[Query] job_id=%s, temp_table=%s.%s, range=[%s, %s)', ++ job_id, ++ self._temp_dataset, ++ temp_table_id, ++ _utc(qr.chunk_start), ++ _utc(qr.chunk_end)) ++ ++ temp_table_ref = bigquery.TableReference( ++ projectId=self._project, ++ datasetId=self._temp_dataset, ++ tableId=temp_table_id) ++ ++ reference = bigquery.JobReference( ++ jobId=job_id, projectId=self._project, location=self._location) ++ ++ request = bigquery.BigqueryJobsInsertRequest( ++ projectId=self._project, ++ job=bigquery.Job( ++ configuration=bigquery.JobConfiguration( ++ query=bigquery.JobConfigurationQuery( ++ query=sql, ++ useLegacySql=False, ++ destinationTable=temp_table_ref, ++ writeDisposition='WRITE_TRUNCATE', ++ ), ++ ), ++ jobReference=reference)) ++ ++ _LOGGER.info('[Query] Submitting BQ job %s...', job_id) ++ response = self._bq_wrapper._start_job(request) ++ _LOGGER.info('[Query] BQ job %s submitted, waiting...', job_id) ++ self._bq_wrapper.wait_for_bq_job( ++ response.jobReference, sleep_duration_sec=2) ++ _LOGGER.info( ++ '[Query] BQ job %s DONE. Results in %s.%s', ++ job_id, ++ self._temp_dataset, ++ temp_table_id) ++ Metrics.counter('BigQueryChangeHistory', 'queries').inc() ++ ++ yield _QueryResult( ++ temp_table_ref=temp_table_ref, ++ range_start=qr.chunk_start, ++ range_end=qr.chunk_end) ++ ++ ++class _CDCWatermarkEstimatorProvider(WatermarkEstimatorProvider): ++ """WatermarkEstimatorProvider that initializes the hold from _QueryResult. ++ ++ Uses range_start from the element to set the initial watermark hold. ++ This prevents the runner from advancing the watermark past the data's ++ timestamps before any rows are emitted. ++ """ ++ def initial_estimator_state( ++ self, element: _QueryResult, ++ restriction: _StreamRestriction) -> Timestamp: ++ return element.range_start ++ ++ def create_watermark_estimator( ++ self, estimator_state: Timestamp) -> ManualWatermarkEstimator: ++ return ManualWatermarkEstimator(estimator_state) ++ ++ ++# ============================================================================= ++# Read: _ReadStorageStreamsSDF ++# ============================================================================= ++ ++ ++class _ReadStorageStreamsSDF(beam.DoFn, ++ beam.transforms.core.RestrictionProvider): ++ """SDF that reads a temp table via BigQuery Storage Read API. ++ ++ Note on SDF lifecycle: the runner decomposes this SDF into three internal ++ wrapper DoFns, each a separately deserialized copy: ++ - Stage A (PairWithRestriction): calls initial_restriction(): no setup() ++ - Stage B (SplitAndSizeRestrictions): calls split(), restriction_size() ++ - Stage C (ProcessSizedElements): calls setup(), then process() ++ Because initial_restriction() runs on a different copy than process(), ++ _ensure_client() lazily creates a gRPC client on whichever copy needs one. ++ The _StreamRestriction carries stream names directly so no shared state ++ is needed between copies. ++ ++ Each element is a _QueryResult pointing to a temp table. ++ ++ Watermark: Uses ManualWatermarkEstimator so the watermark only advances ++ as fast as the change-timestamp values we emit. ++ ++ Emits: ++ Main output: TimestampedValue(row_dict, event_timestamp) ++ Side output (_CLEANUP_TAG): (table_key, (streams_read, total_streams)) ++ """ ++ def __init__( ++ self, ++ batch_arrow_read: bool = True, ++ change_timestamp_column: str = 'change_timestamp', ++ max_split_rounds: int = 1, ++ emit_raw_batches: bool = False) -> None: ++ self._batch_arrow_read = batch_arrow_read ++ self._change_timestamp_column = change_timestamp_column ++ self._max_split_rounds = max_split_rounds ++ self._emit_raw_batches = emit_raw_batches ++ self._storage_client = None ++ ++ def _ensure_client(self) -> None: ++ """Lazily initialize the Storage client. ++ ++ Called from both setup() and initial_restriction() because the runner ++ may invoke initial_restriction on the RestrictionProvider instance ++ before setup() runs (or on a separately deserialized copy). ++ """ ++ if self._storage_client is None: ++ _LOGGER.info('[Read] creating BigQueryReadClient') ++ self._storage_client = bq_storage.BigQueryReadClient() ++ ++ def setup(self) -> None: ++ self._ensure_client() ++ ++ def _split_all_streams(self, stream_names: Tuple[str, ...], ++ max_split_rounds: int) -> Tuple[str, ...]: ++ """Split each stream at fraction=0.5 for up to max_split_rounds rounds. ++ ++ Each round attempts to split every stream in the current list. A ++ successful split replaces the original stream with primary + remainder. ++ A refused split (both fields empty) keeps the original stream intact. ++ Stops when max_split_rounds is reached or a full round produces zero ++ new splits. ++ ++ BQ's server-side granularity controls how many splits are possible. ++ Small tables may not split at all; large tables may allow multiple ++ rounds of doubling. ++ """ ++ result = list(stream_names) ++ for round_num in range(1, max_split_rounds + 1): ++ new_result = [] ++ made_progress = False ++ for name in result: ++ response = self._storage_client.split_read_stream( ++ request=bq_storage.types.SplitReadStreamRequest( ++ name=name, fraction=0.5)) ++ primary = response.primary_stream.name ++ remainder = response.remainder_stream.name ++ if primary and remainder: ++ new_result.extend([primary, remainder]) ++ made_progress = True ++ else: ++ new_result.append(name) ++ result = new_result ++ _LOGGER.info( ++ '[Read] _split_all_streams round %d/%d: %d streams ' ++ '(progress=%s)', ++ round_num, ++ max_split_rounds, ++ len(result), ++ made_progress) ++ if not made_progress: ++ break ++ return tuple(result) ++ ++ def initial_restriction(self, element: _QueryResult) -> _StreamRestriction: ++ """Create ReadSession and return _StreamRestriction with stream names. ++ ++ When max_split_rounds > 0, uses SplitReadStream to subdivide each ++ stream at fraction=0.5 for up to max_split_rounds rounds, maximizing ++ parallelism beyond what CreateReadSession provides. ++ """ ++ self._ensure_client() ++ table_key = _table_key(element.temp_table_ref) ++ session = self._create_read_session(element.temp_table_ref) ++ stream_names = tuple(s.name for s in session.streams) ++ original_count = len(stream_names) ++ _LOGGER.info( ++ '[Read] initial_restriction for %s: %d streams from CreateReadSession', ++ table_key, ++ original_count) ++ ++ if self._max_split_rounds > 0: ++ stream_names = self._split_all_streams( ++ stream_names, self._max_split_rounds) ++ _LOGGER.info( ++ '[Read] initial_restriction for %s: %d -> %d streams ' ++ 'after SplitReadStream', ++ table_key, ++ original_count, ++ len(stream_names)) ++ ++ return _StreamRestriction(stream_names, 0, len(stream_names)) ++ ++ def create_tracker( ++ self, restriction: _StreamRestriction) -> _StreamRestrictionTracker: ++ return _StreamRestrictionTracker(restriction) ++ ++ def restriction_size( ++ self, element: _QueryResult, restriction: _StreamRestriction) -> int: ++ return restriction.size() ++ ++ def split(self, element: _QueryResult, ++ restriction: _StreamRestriction) -> Iterable[_StreamRestriction]: ++ """Yield one _StreamRestriction per stream for parallel distribution.""" ++ if restriction.size() <= 1: ++ yield restriction ++ else: ++ for i in range(restriction.start, restriction.stop): ++ yield _StreamRestriction(restriction.stream_names, i, i + 1) ++ ++ def is_bounded(self) -> bool: ++ return True ++ ++ def process( ++ self, ++ element: _QueryResult, ++ restriction_tracker=beam.DoFn.RestrictionParam(), ++ watermark_estimator=beam.DoFn.WatermarkEstimatorParam( ++ _CDCWatermarkEstimatorProvider()) ++ ): ++ self._ensure_client() ++ table_key = _table_key(element.temp_table_ref) ++ ++ _LOGGER.info( ++ '[Read] Processing %s, range=[%s, %s), ' ++ 'initial watermark=%s', ++ table_key, ++ _utc(element.range_start), ++ _utc(element.range_end), ++ _utc(watermark_estimator.current_watermark())) ++ ++ restriction = restriction_tracker.current_restriction() ++ stream_names = restriction.stream_names ++ total_streams = len(stream_names) ++ ++ streams_read = 0 ++ ++ _LOGGER.info( ++ '[Read] Reading streams [%d, %d) of %d total for %s', ++ restriction.start, ++ restriction.stop, ++ total_streams, ++ table_key) ++ ++ for i in range(restriction.start, restriction.stop): ++ if not restriction_tracker.try_claim(i): ++ _LOGGER.info( ++ '[Read] try_claim(%d) FAILED for %s: ' ++ 'runner split or checkpoint, breaking', ++ i, ++ table_key) ++ break ++ ++ stream_name = stream_names[i] ++ _LOGGER.info( ++ '[Read] try_claim(%d) succeeded: reading stream %s', i, stream_name) ++ ++ if self._emit_raw_batches: ++ stream_batches = 0 ++ for raw_batch in self._read_stream_raw(stream_name): ++ yield TimestampedValue(raw_batch, element.range_start) ++ stream_batches += 1 ++ Metrics.counter( ++ 'BigQueryChangeHistory', 'batches_emitted').inc(stream_batches) ++ _LOGGER.info( ++ '[Read] Finished reading stream %d for %s: %d batches', ++ i, table_key, stream_batches) ++ else: ++ stream_rows = 0 ++ for row in self._read_stream(stream_name): ++ ts = row.get(self._change_timestamp_column) ++ if ts is None: ++ raise ValueError( ++ 'Row missing %r column. Row keys: %s' % ++ (self._change_timestamp_column, list(row.keys()))) ++ if isinstance(ts, datetime.datetime): ++ ts = Timestamp.from_utc_datetime(ts) ++ ++ yield TimestampedValue(row, ts) ++ stream_rows += 1 ++ Metrics.counter( ++ 'BigQueryChangeHistory', 'rows_emitted').inc(stream_rows) ++ _LOGGER.info( ++ '[Read] Finished reading stream %d for %s: %d rows', ++ i, table_key, stream_rows) ++ ++ streams_read += 1 ++ Metrics.counter('BigQueryChangeHistory', 'streams_read').inc() ++ ++ # Advance watermark to range_end after reading all streams. The ++ # initial hold was set to range_start by _CDCWatermarkEstimatorProvider. ++ watermark_estimator.set_watermark(element.range_end) ++ _LOGGER.info( ++ '[Read] Watermark advanced to %s (range_end) for %s', ++ _utc(element.range_end), ++ table_key) ++ ++ # Release the storage client so the gRPC channel doesn't go stale ++ # between process() calls. _ensure_client() will create a fresh one. ++ self._storage_client = None ++ ++ # Emit cleanup signal. Every split that reads at least one stream ++ # reports how many it read. ++ if streams_read > 0: ++ _LOGGER.info( ++ '[Read] Emitting cleanup signal for %s: ' ++ 'streams_read=%d, total_streams=%d', ++ table_key, ++ streams_read, ++ total_streams) ++ yield beam.pvalue.TaggedOutput( ++ _CLEANUP_TAG, (table_key, (streams_read, total_streams))) ++ ++ def _create_read_session(self, table_ref: 'bigquery.TableReference') -> Any: ++ """Create a BigQuery Storage ReadSession for the given table.""" ++ table_path = ( ++ f'projects/{table_ref.projectId}/' ++ f'datasets/{table_ref.datasetId}/' ++ f'tables/{table_ref.tableId}') ++ ++ requested_session = bq_storage.types.ReadSession() ++ requested_session.table = table_path ++ requested_session.data_format = bq_storage.types.DataFormat.ARROW ++ read_options = requested_session.read_options ++ read_options.arrow_serialization_options.buffer_compression = ( ++ bq_storage.types.ArrowSerializationOptions.CompressionCodec.ZSTD) ++ ++ session = self._storage_client.create_read_session( ++ parent=f'projects/{table_ref.projectId}', ++ read_session=requested_session, ++ max_stream_count=_DEFAULT_MAX_STREAMS) ++ _LOGGER.info( ++ '[Read] _create_read_session: table=%s, %d streams', ++ table_path, ++ len(session.streams)) ++ return session ++ ++ def _read_stream(self, stream_name: str) -> Iterable[Dict[str, Any]]: ++ """Read all rows from a single Storage API stream as dicts. ++ ++ When batch_arrow_read is enabled, converts entire Arrow RecordBatches ++ at once using to_pylist() instead of calling .as_py() on each cell ++ individually. This is ~1.5x faster for large tables at the cost of ~2x ++ peak memory per batch. ++ """ ++ if self._batch_arrow_read: ++ yield from self._read_stream_batch(stream_name) ++ else: ++ yield from self._read_stream_row_by_row(stream_name) ++ ++ def _read_stream_row_by_row(self, ++ stream_name: str) -> Iterable[Dict[str, Any]]: ++ """Row-by-row Arrow conversion (lower memory than batch mode).""" ++ t0 = time.time() ++ row_count = 0 ++ for row in self._storage_client.read_rows(stream_name).rows(): ++ yield dict((item[0], item[1].as_py()) for item in row.items()) ++ row_count += 1 ++ elapsed = time.time() - t0 ++ _LOGGER.info( ++ '[Read] row_by_row: %d rows in %.2fs (%.0f rows/s)', ++ row_count, ++ elapsed, ++ row_count / elapsed if elapsed > 0 else 0) ++ ++ def _read_stream_batch(self, stream_name: str) -> Iterable[Dict[str, Any]]: ++ """Batch-convert Arrow RecordBatches for high throughput.""" ++ schema = None ++ row_count = 0 ++ t0 = time.time() ++ for response in self._storage_client.read_rows(stream_name): ++ if schema is None and response.arrow_schema.serialized_schema: ++ schema = pyarrow.ipc.read_schema( ++ pyarrow.py_buffer(response.arrow_schema.serialized_schema)) ++ batch_bytes = response.arrow_record_batch.serialized_record_batch ++ if batch_bytes and schema is not None: ++ batch = pyarrow.ipc.read_record_batch( ++ pyarrow.py_buffer(batch_bytes), schema) ++ yield from batch.to_pylist() ++ row_count += batch.num_rows ++ elapsed = time.time() - t0 ++ _LOGGER.info( ++ '[Read] batch_read: %d rows in %.2fs (%.0f rows/s)', ++ row_count, ++ elapsed, ++ row_count / elapsed if elapsed > 0 else 0) ++ ++ def _read_stream_raw( ++ self, ++ stream_name: str) -> Iterable[Tuple[bytes, bytes]]: ++ """Yield raw (schema_bytes, batch_bytes) without decompression. ++ ++ Used when emit_raw_batches is enabled to defer decompression and ++ Arrow-to-Python conversion to a downstream DoFn after reshuffling. ++ Schema bytes are included in each tuple so each batch is ++ self-contained and can be decoded independently. ++ """ ++ schema_bytes = b'' ++ batch_count = 0 ++ t0 = time.time() ++ for response in self._storage_client.read_rows(stream_name): ++ if not schema_bytes and response.arrow_schema.serialized_schema: ++ schema_bytes = bytes(response.arrow_schema.serialized_schema) ++ batch_bytes = response.arrow_record_batch.serialized_record_batch ++ if batch_bytes and schema_bytes: ++ yield (schema_bytes, bytes(batch_bytes)) ++ batch_count += 1 ++ elapsed = time.time() - t0 ++ _LOGGER.info( ++ '[Read] raw_read: %d batches in %.2fs', ++ batch_count, ++ elapsed) ++ ++ ++class _DecompressArrowBatchesFn(beam.DoFn): ++ """Decompress and convert raw Arrow batches to timestamped row dicts. ++ ++ Receives GBK output: (shard_key, Iterable[(schema_bytes, batch_bytes)]) ++ and converts each batch to individual row dicts with event timestamps ++ extracted from the change_timestamp column. ++ """ ++ def __init__(self, change_timestamp_column: str = 'change_timestamp') -> None: ++ self._change_timestamp_column = change_timestamp_column ++ ++ def process( ++ self, ++ element: Tuple[int, Iterable[Tuple[bytes, bytes]]] ++ ) -> Iterable[Dict[str, Any]]: ++ _, batches = element ++ for schema_bytes, batch_bytes in batches: ++ schema = pyarrow.ipc.read_schema(pyarrow.py_buffer(schema_bytes)) ++ batch = pyarrow.ipc.read_record_batch( ++ pyarrow.py_buffer(batch_bytes), schema) ++ ++ rows = batch.to_pylist() ++ for row in rows: ++ ts = row.get(self._change_timestamp_column) ++ if ts is None: ++ raise ValueError( ++ 'Row missing %r column. Row keys: %s' % ++ (self._change_timestamp_column, list(row.keys()))) ++ if isinstance(ts, datetime.datetime): ++ ts = Timestamp.from_utc_datetime(ts) ++ yield TimestampedValue(row, ts) ++ Metrics.counter('BigQueryChangeHistory', 'rows_emitted').inc(len(rows)) ++ ++ ++# ============================================================================= ++# Cleanup: _CleanupTempTablesFn ++# ============================================================================= ++ ++ ++class _CleanupTempTablesFn(beam.DoFn): ++ """Stateful DoFn that deletes temp tables after all streams are read. ++ ++ Receives cleanup signals from the Read SDF as: ++ (table_key, (streams_read_count, total_streams)) ++ ++ Accumulates streams_read across all signals for the same table_key. ++ When streams_read >= total_streams, deletes the temp table. The >= ++ (rather than ==) guards against duplicate delivery in at-least-once runners. ++ """ ++ STREAMS_READ = beam.transforms.userstate.CombiningValueStateSpec( ++ 'streams_read', sum) ++ ++ def setup(self) -> None: ++ _LOGGER.info('[Cleanup] setup: creating BigQueryWrapper') ++ self._bq_wrapper = bigquery_tools.BigQueryWrapper() ++ ++ def process( ++ self, ++ element: Tuple[str, Tuple[int, int]], ++ streams_read=beam.DoFn.StateParam(STREAMS_READ) ++ ) -> None: ++ table_key = element[0] ++ split_count = element[1][0] ++ total_streams = element[1][1] ++ ++ _LOGGER.info( ++ '[Cleanup] Received cleanup signal for %s: ' ++ 'split_count=%d, total_streams=%d', ++ table_key, ++ split_count, ++ total_streams) ++ ++ streams_read.add(split_count) ++ current_read = streams_read.read() ++ ++ _LOGGER.info( ++ '[Cleanup] State for %s: streams_read=%d/%d', ++ table_key, ++ current_read, ++ total_streams) ++ ++ if current_read >= total_streams: ++ parts = table_key.split('.') ++ if len(parts) == 3: ++ project, dataset, table = parts ++ _LOGGER.info( ++ '[Cleanup] All streams read: DELETING temp table %s', table_key) ++ self._bq_wrapper._delete_table(project, dataset, table) ++ _LOGGER.info('[Cleanup] Deleted temp table %s', table_key) ++ Metrics.counter('BigQueryChangeHistory', 'temp_tables_deleted').inc() ++ streams_read.clear() ++ else: ++ _LOGGER.info( ++ '[Cleanup] Not yet complete for %s (%d/%d), ' ++ 'waiting for more signals', ++ table_key, ++ current_read, ++ total_streams) ++ ++ ++# ============================================================================= ++# Public API: ReadBigQueryChangeHistory ++# ============================================================================= ++ ++ ++class ReadBigQueryChangeHistory(beam.PTransform): ++ """Streaming source for BigQuery change history. ++ ++ Continuously polls BigQuery APPENDS() or CHANGES() functions and emits ++ changed rows as an unbounded PCollection of dicts. ++ ++ Args: ++ table: BigQuery table to read changes from. ++ Format: 'project:dataset.table' or 'project.dataset.table'. ++ poll_interval_sec: Seconds between polls. Default 60. ++ start_time: Start reading from this timestamp (float, epoch seconds). ++ Default: current time when pipeline starts. ++ stop_time: Stop polling at this timestamp. Default: run forever. ++ change_function: 'CHANGES' or 'APPENDS'. Default 'APPENDS'. ++ buffer_sec: Safety buffer in seconds behind now(). Default 10. BQ does not ++ fail or wait if the query end_ts is less than BQ's CURRENT_TIMESTAMP. ++ This is an extra guardrail to protect against silent data. ++ project: GCP project ID. Default: from pipeline options. ++ temp_dataset: Dataset for temp tables. If None (default), a ++ per-pipeline dataset is auto-created with a 24-hour table ++ expiration as a safety net for orphaned tables. Set this to ++ use an existing dataset (e.g. if your service account lacks ++ bigquery.datasets.create permission). ++ location: BigQuery geographic location for query jobs and temp ++ dataset (e.g. 'US', 'us-central1'). If None (default), inferred ++ from the source table. ++ change_type_column: Output column name for the _CHANGE_TYPE ++ pseudo-column. Default 'change_type'. Change this if your source ++ table already has a column named 'change_type'. ++ change_timestamp_column: Output column name for the ++ _CHANGE_TIMESTAMP pseudo-column. Default 'change_timestamp'. ++ Change this if your source table already has a column named ++ 'change_timestamp'. This column is also used internally to ++ extract event timestamps for watermark tracking. ++ columns: Optional list of column names to select from the source ++ table. If None (default), all columns are selected. The ++ pseudo-columns (change_type, change_timestamp) are always ++ included regardless of this setting. ++ row_filter: Optional SQL boolean expression used as a WHERE clause ++ on the CHANGES/APPENDS query. Do not include the WHERE keyword. ++ Example: ``'status = "active" AND region = "US"'``. ++ batch_arrow_read: If True (default), convert Arrow RecordBatches in ++ bulk using to_pylist() instead of per-cell .as_py() calls. ++ This is 1.5x faster for large tables at the cost of ~2x peak ++ memory per RecordBatch. Set to False for minimal memory usage. ++ max_split_rounds: Maximum number of recursive SplitReadStream ++ rounds. Each round splits every stream at fraction=0.5, ++ potentially doubling the stream count (if BQ allows). Default ++ 1 (one round of splitting). Set 0 to disable splitting ++ entirely. Set higher for very large tables where more ++ parallelism is needed. ++ decompress_shards: If set to a positive integer, the Read SDF ++ emits raw compressed Arrow batches instead of decoded rows. ++ The batches are reshuffled for fan-out and then decoded in a ++ separate DoFn. This spreads decompression and Arrow-to-Python ++ conversion CPU across more workers. If None (default), rows ++ are decoded inline within the Read SDF. ++ """ ++ def __init__( ++ self, ++ table: str, ++ poll_interval_sec: float = 60, ++ start_time: Optional[float] = None, ++ stop_time: Optional[float] = None, ++ change_function: str = 'APPENDS', ++ buffer_sec: float = 10, ++ project: Optional[str] = None, ++ temp_dataset: Optional[str] = None, ++ location: Optional[str] = None, ++ change_type_column: str = 'change_type', ++ change_timestamp_column: str = 'change_timestamp', ++ columns: Optional[List[str]] = None, ++ row_filter: Optional[str] = None, ++ batch_arrow_read: bool = True, ++ max_split_rounds: int = 1, ++ decompress_shards: Optional[int] = None) -> None: ++ super().__init__() ++ if bq_storage is None: ++ raise ImportError( ++ 'google-cloud-bigquery-storage is required for ' ++ 'ReadBigQueryChangeHistory. Install it with: ' ++ 'pip install google-cloud-bigquery-storage') ++ if pyarrow is None: ++ raise ImportError( ++ 'pyarrow is required for ReadBigQueryChangeHistory. ' ++ 'Install it with: pip install pyarrow') ++ if change_function not in ('CHANGES', 'APPENDS'): ++ raise ValueError( ++ f"change_function must be 'CHANGES' or 'APPENDS', " ++ f"got '{change_function}'") ++ if poll_interval_sec < 15: ++ raise ValueError( ++ f'poll_interval_sec must be >= 15, got {poll_interval_sec}') ++ if buffer_sec < 0: ++ raise ValueError(f'buffer_sec must be >= 0, got {buffer_sec}') ++ self._table = table ++ self._poll_interval_sec = poll_interval_sec ++ self._start_time = start_time ++ self._stop_time = stop_time ++ self._change_function = change_function ++ self._buffer_sec = buffer_sec ++ self._project = project ++ self._temp_dataset = temp_dataset ++ self._location = location ++ self._change_type_column = change_type_column ++ self._change_timestamp_column = change_timestamp_column ++ self._columns = columns ++ self._row_filter = row_filter ++ self._batch_arrow_read = batch_arrow_read ++ self._max_split_rounds = max_split_rounds ++ self._decompress_shards = decompress_shards ++ ++ def expand(self, pbegin: beam.pvalue.PBegin) -> beam.PCollection: ++ project = self._project ++ if project is None: ++ project = pbegin.pipeline.options.view_as( ++ beam.options.pipeline_options.GoogleCloudOptions).project ++ ++ if project is None: ++ raise ValueError( ++ 'project must be specified either in ReadBigQueryChangeHistory ' ++ 'or in pipeline options (--project)') ++ ++ start_time = Timestamp(self._start_time or time.time()) ++ stop_time = ( ++ Timestamp(self._stop_time) ++ if self._stop_time is not None else MAX_TIMESTAMP) ++ buffer = Duration(seconds=self._buffer_sec) ++ poll_interval = Duration(seconds=self._poll_interval_sec) ++ ++ temp_dataset = self._temp_dataset ++ if temp_dataset is None: ++ temp_dataset = f'beam_ch_temp_{uuid.uuid4().hex[:12]}' ++ ++ _LOGGER.info( ++ '[ReadBigQueryChangeHistory] expand: table=%s, project=%s, ' ++ 'change_function=%s, poll_interval=%d sec, buffer=%d sec, ' ++ 'temp_dataset=%s, start_time=%s, stop_time=%s', ++ self._table, ++ project, ++ self._change_function, ++ self._poll_interval_sec, ++ self._buffer_sec, ++ temp_dataset, ++ _utc(start_time), ++ _utc(stop_time) if stop_time != MAX_TIMESTAMP else 'INF') ++ ++ # Custom polling SDF emits lightweight _QueryRange instructions. ++ # The SDF uses defer_remainder() for poll timing and ++ # _PollWatermarkEstimator to hold the watermark at data timestamps. ++ # On the first invocation it handles the full historical range ++ # [start_time, now - buffer) in a single poll. ++ config = _PollConfig(start_time=start_time) ++ ++ query_ranges = ( ++ pbegin ++ | 'CreatePollConfig' >> beam.Create([config]) ++ | 'PollChangeHistory' >> beam.ParDo( ++ _PollChangeHistoryFn( ++ table=self._table, ++ project=project, ++ change_function=self._change_function, ++ buffer=buffer, ++ start_time=start_time, ++ stop_time=stop_time, ++ poll_interval=poll_interval, ++ location=self._location))) ++ ++ # CommitQueryResults: Reshuffle commits _QueryResult (temp table ref) ++ # so that if the Read SDF retries, it re-reads the existing temp table ++ # instead of re-running the BQ query. ++ # Possible edge-case is that if ReadStorageStreams doesn't read the temp ++ # table within 24 hours (table expiration) it can end up in a bad state by ++ # trying to query a non-existing table. ++ query_results = ( ++ query_ranges ++ | 'CommitQueryRanges' >> beam.Reshuffle() ++ | 'ExecuteQueries' >> beam.ParDo( ++ _ExecuteQueryFn( ++ table=self._table, ++ project=project, ++ change_function=self._change_function, ++ temp_dataset=temp_dataset, ++ location=self._location, ++ change_type_column=self._change_type_column, ++ change_timestamp_column=self._change_timestamp_column, ++ columns=self._columns, ++ row_filter=self._row_filter)) ++ | 'CommitQueryResults' >> beam.Reshuffle()) ++ ++ emit_raw = self._decompress_shards is not None ++ ++ read_sdf = beam.ParDo( ++ _ReadStorageStreamsSDF( ++ batch_arrow_read=self._batch_arrow_read, ++ change_timestamp_column=self._change_timestamp_column, ++ max_split_rounds=self._max_split_rounds, ++ emit_raw_batches=emit_raw)) ++ if emit_raw: ++ read_sdf = read_sdf.with_output_types(Tuple[bytes, bytes]) ++ else: ++ read_sdf = read_sdf.with_output_types(Dict[str, Any]) ++ ++ read_outputs = ( ++ query_results ++ | 'ReadStorageStreams' >> read_sdf.with_outputs( ++ _CLEANUP_TAG, main='rows')) ++ ++ _ = ( ++ read_outputs[_CLEANUP_TAG] ++ | 'CleanupTempTables' >> beam.ParDo(_CleanupTempTablesFn())) ++ ++ if emit_raw: ++ # Fan out raw Arrow batches across decompress_shards workers ++ # via GBK, then decompress and convert to timestamped row dicts. ++ # Uses a discarding trigger so GBK fires per-element without ++ # waiting for the GlobalWindow to close. ++ num_shards = self._decompress_shards ++ rows = ( ++ read_outputs['rows'] ++ | 'ShardBatches' >> beam.WithKeys( ++ lambda _, n=num_shards: random.randint(0, n - 1)) ++ | 'WindowForGBK' >> beam.WindowInto( ++ GlobalWindows(), ++ trigger=beam_trigger.Repeatedly( ++ beam_trigger.AfterCount(1)), ++ accumulation_mode=( ++ beam_trigger.AccumulationMode.DISCARDING)) ++ | 'GroupByShardKey' >> beam.GroupByKey() ++ | 'DecompressBatches' >> beam.ParDo( ++ _DecompressArrowBatchesFn( ++ change_timestamp_column=( ++ self._change_timestamp_column)))) ++ return rows ++ else: ++ return read_outputs['rows'] +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py +new file mode 100644 +index 000000000..3b7cc8025 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py +@@ -0,0 +1,632 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Configurable metric computation for anomaly detection pipelines. ++ ++This module provides a ``MetricSpec`` configuration system and a ++``ComputeMetric`` PTransform that computes windowed, grouped metrics from ++raw row dicts (e.g., from ``ReadBigQueryChangeHistory``). The output is ++suitable for feeding directly into ``AnomalyDetection``. ++ ++Example usage:: ++ ++ from bqmonitor.metric import ( ++ MetricSpec, AggregationSpec, WindowSpec, MeasureSpec, ++ DerivedField, WindowType, AggOp, ComputeMetric) ++ from bqmonitor.safe_eval import Expr ++ from apache_beam.ml.anomaly.transforms import AnomalyDetection ++ from apache_beam.ml.anomaly.detectors.zscore import ZScore ++ ++ # CUJ 1: Total revenue per hour ++ spec = MetricSpec( ++ name='revenue', ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=3600), ++ measures=[MeasureSpec( ++ field='transaction_amount', agg=AggOp.SUM, alias='revenue')], ++ ), ++ ) ++ result = cdc_rows | ComputeMetric(spec) | AnomalyDetection(ZScore()) ++ ++ # CUJ 2: CTR grouped by dimensions ++ spec = MetricSpec( ++ name='ctr', ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=86400), ++ group_by=['campaign_type', 'user_segment'], ++ measures=[ ++ MeasureSpec(field='is_click', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec(field='is_click', agg=AggOp.COUNT, ++ alias='impressions'), ++ ], ++ ), ++ measure_combiner=Expr.from_string("clicks / impressions"), ++ ) ++ ++ # CUJ 3: Success rate with derived field ++ spec = MetricSpec( ++ name='success_rate', ++ derived_fields=[ ++ DerivedField( ++ name='is_success', ++ expression=Expr.from_string( ++ "1 if status == 'success' else 0")), ++ ], ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=86400), ++ group_by=['brand_name', 'category'], ++ measures=[ ++ MeasureSpec(field='is_success', agg=AggOp.SUM, ++ alias='successes'), ++ MeasureSpec(field='is_success', agg=AggOp.COUNT, alias='total'), ++ ], ++ ), ++ measure_combiner=Expr.from_string("successes / total"), ++ ) ++""" ++ ++import dataclasses ++import random ++from enum import Enum ++from typing import Any ++from typing import Optional ++from typing import Tuple ++ ++import apache_beam as beam ++from apache_beam.transforms import combiners ++from apache_beam.transforms import window as beam_window ++ ++from bqmonitor.safe_eval import Expr ++from apache_beam.ml.anomaly.specifiable import specifiable ++ ++ ++class WindowType(Enum): ++ """Window type for metric aggregation.""" ++ FIXED = 'fixed' ++ SLIDING = 'sliding' ++ ++ ++class AggOp(Enum): ++ """Aggregation operator.""" ++ SUM = 'SUM' ++ COUNT = 'COUNT' ++ MIN = 'MIN' ++ MAX = 'MAX' ++ MEAN = 'MEAN' ++ ++ ++class FanoutStrategy(Enum): ++ """Strategy for global (non-keyed) aggregation parallelism. ++ ++ NONE: Plain CombineGlobally, no fanout. Relies on combiner lifting (PGBK) ++ for mapper-side pre-combining. Works well when upstream provides enough ++ parallel bundles (e.g. decompress_shards) and streaming state I/O on a ++ single key is not a bottleneck. ++ HOTKEY_FANOUT: Uses CombineGlobally.with_fanout(). Per-bundle nonce sharding. ++ Better PGBK table efficiency (1 slot per bundle), but shard distribution ++ depends on bundle count and sizes. Good for multi-key CombinePerKey where ++ only a few keys are hot. ++ SHARDED: Per-element random sharding with _PreCombineFn/_PostCombineFn. ++ Uniform distribution regardless of bundle count. One extra GBK vs NONE, ++ but distributes streaming state I/O across shard keys. Best for single ++ global key at high throughput. ++ """ ++ NONE = 'none' ++ HOTKEY_FANOUT = 'hotkey_fanout' ++ SHARDED = 'sharded' ++ ++ ++@dataclasses.dataclass(frozen=True) ++class WindowSpec: ++ """Window configuration for metric aggregation. ++ ++ Args: ++ type: FIXED or SLIDING window. ++ size_seconds: Window size in seconds. ++ period_seconds: Slide period in seconds (required for SLIDING, ignored for ++ FIXED). ++ """ ++ type: WindowType = WindowType.FIXED ++ size_seconds: int = 3600 ++ period_seconds: Optional[int] = None ++ ++ ++@dataclasses.dataclass(frozen=True) ++class DerivedField: ++ """Pre-aggregation column derivation via expression. ++ ++ Args: ++ name: Name of the new field to create. ++ expression: A compiled ``Expr`` callable, e.g. ++ ``Expr.from_string("1 if status == 'success' else 0")``. ++ """ ++ name: str ++ expression: Expr ++ ++ ++@dataclasses.dataclass(frozen=True) ++class MeasureSpec: ++ """A single aggregation measure. ++ ++ Args: ++ field: Input field name to aggregate. ++ agg: The aggregation operator. ++ alias: Output name for this measure's result. ++ """ ++ field: str ++ agg: AggOp ++ alias: str ++ ++ ++@dataclasses.dataclass(frozen=True) ++class AggregationSpec: ++ """Windowed grouped aggregation configuration. ++ ++ Args: ++ window: Window configuration. ++ group_by: Field names for grouping. Empty list means global aggregation. ++ measures: List of aggregation measures. ++ """ ++ window: WindowSpec = dataclasses.field(default_factory=WindowSpec) ++ group_by: list = dataclasses.field(default_factory=list) ++ measures: list = dataclasses.field(default_factory=list) ++ ++ ++@specifiable ++class MetricSpec: ++ """Complete metric computation specification. ++ ++ Defines how to transform raw row dicts into a single numeric metric value ++ suitable for anomaly detection. ++ ++ Args: ++ aggregation: Windowed grouped aggregation spec. ++ derived_fields: Optional pre-aggregation derived fields. ++ measure_combiner: Optional post-aggregation ``Expr`` operating on measure ++ aliases. Required when there are multiple measures. ++ name: Optional human-readable metric name. ++ """ ++ def __init__( ++ self, ++ aggregation, ++ derived_fields=None, ++ measure_combiner=None, ++ name=None, ++ ): ++ self.name = name ++ self.aggregation = aggregation ++ self.derived_fields = derived_fields or [] ++ self.measure_combiner = measure_combiner ++ self._validate() ++ ++ def _validate(self): ++ agg = self.aggregation ++ if not agg.measures: ++ raise ValueError("MetricSpec requires at least one measure") ++ if self.measure_combiner is None and len(agg.measures) > 1: ++ raise ValueError( ++ "measure_combiner is required when there are multiple measures. " ++ f"Got {len(agg.measures)} measures: " ++ f"{[m.alias for m in agg.measures]}") ++ if (agg.window.type == WindowType.SLIDING and ++ agg.window.period_seconds is None): ++ raise ValueError("period_seconds is required for SLIDING windows") ++ for df in self.derived_fields: ++ if not isinstance(df.expression, Expr): ++ raise TypeError( ++ f"DerivedField.expression must be an Expr, " ++ f"got {type(df.expression).__name__}") ++ if (self.measure_combiner is not None and ++ not isinstance(self.measure_combiner, Expr)): ++ raise TypeError( ++ f"measure_combiner must be an Expr, " ++ f"got {type(self.measure_combiner).__name__}") ++ # Validate that measure_combiner only references known measure aliases. ++ if self.measure_combiner is not None: ++ aliases = {m.alias for m in agg.measures} ++ unknown = self.measure_combiner.field_refs() - aliases ++ if unknown: ++ raise ValueError( ++ f"measure_combiner references unknown fields: {unknown}. " ++ f"Available measure aliases: {aliases}") ++ ++ def required_source_columns(self): ++ """Return the set of source table columns needed by this metric spec. ++ ++ This includes group_by fields, measure fields (excluding derived field ++ names), and field references from derived field expressions. ++ """ ++ derived_names = {df.name for df in self.derived_fields} ++ cols = set() ++ cols.update(self.aggregation.group_by) ++ for m in self.aggregation.measures: ++ if m.agg != AggOp.COUNT and m.field not in derived_names: ++ cols.add(m.field) ++ for df in self.derived_fields: ++ cols.update(df.expression.field_refs()) ++ return cols ++ ++ def to_dict(self): ++ """Serialize to a plain dict suitable for JSON.""" ++ result = { ++ 'aggregation': { ++ 'window': { ++ 'type': self.aggregation.window.type.value, ++ 'size_seconds': self.aggregation.window.size_seconds, ++ 'period_seconds': self.aggregation.window.period_seconds, ++ }, ++ 'group_by': list(self.aggregation.group_by), ++ 'measures': [{ ++ 'field': m.field, 'agg': m.agg.value, 'alias': m.alias ++ } for m in self.aggregation.measures], ++ }, ++ } ++ if self.derived_fields: ++ result['derived_fields'] = [{ ++ 'name': df.name, 'expression': str(df.expression) ++ } for df in self.derived_fields] ++ if self.measure_combiner is not None: ++ result['measure_combiner'] = {'expression': str(self.measure_combiner)} ++ if self.name is not None: ++ result['name'] = self.name ++ return result ++ ++ @classmethod ++ def from_dict(cls, d): ++ """Construct a MetricSpec from a plain dict (e.g., loaded from JSON). ++ ++ Expressions (``measure_combiner`` and ``derived_fields[].expression``) ++ are Python expression strings, e.g.:: ++ ++ "measure_combiner": {"expression": "clicks / impressions"} ++ "expression": "1 if status == 'success' else 0" ++ ++ Args: ++ d: Dictionary with keys matching the MetricSpec constructor. ++ ++ Returns: ++ MetricSpec instance. ++ ++ Raises: ++ TypeError: If an expression is not a string. ++ SyntaxError: If an expression string is not valid Python syntax. ++ ValueError: If an expression uses unsupported constructs, or if ++ measure_combiner references fields not in the measure aliases. ++ """ ++ agg_dict = d['aggregation'] ++ window_dict = agg_dict.get('window', {}) ++ window = WindowSpec( ++ type=WindowType(window_dict.get('type', 'fixed')), ++ size_seconds=window_dict.get('size_seconds', 3600), ++ period_seconds=window_dict.get('period_seconds'), ++ ) ++ measures = [ ++ MeasureSpec(field=m['field'], agg=AggOp(m['agg']), alias=m['alias']) ++ for m in agg_dict.get('measures', []) ++ ] ++ derived_fields = None ++ if 'derived_fields' in d and d['derived_fields']: ++ derived_fields = [] ++ for df in d['derived_fields']: ++ expr_val = df['expression'] ++ if not isinstance(expr_val, str): ++ raise TypeError( ++ f"derived_fields[].expression must be a string, " ++ f"got {type(expr_val).__name__}. " ++ f"Example: \"1 if status == 'success' else 0\"") ++ derived_fields.append( ++ DerivedField( ++ name=df['name'], expression=Expr.from_string(expr_val))) ++ measure_combiner = None ++ if 'measure_combiner' in d and d['measure_combiner'] is not None: ++ mc = d['measure_combiner'] ++ expr_val = mc['expression'] if isinstance(mc, dict) else mc ++ if not isinstance(expr_val, str): ++ raise TypeError( ++ f"measure_combiner.expression must be a string, " ++ f"got {type(expr_val).__name__}. " ++ f"Example: \"clicks / impressions\"") ++ measure_combiner = Expr.from_string(expr_val) ++ return cls( ++ aggregation=AggregationSpec( ++ window=window, ++ group_by=agg_dict.get('group_by', []), ++ measures=measures, ++ ), ++ derived_fields=derived_fields, ++ measure_combiner=measure_combiner, ++ name=d.get('name'), ++ _run_init=True, ++ ) ++ ++ ++# --------------------------------------------------------------------------- ++# Internal CombineFn and DoFns ++# --------------------------------------------------------------------------- ++ ++ ++class _SumCombineFn(beam.CombineFn): ++ def create_accumulator(self): ++ return 0 ++ ++ def add_input(self, accumulator, element): ++ return accumulator + element ++ ++ def merge_accumulators(self, accumulators): ++ return sum(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator ++ ++ ++class _MinCombineFn(beam.CombineFn): ++ def create_accumulator(self): ++ return float('inf') ++ ++ def add_input(self, accumulator, element): ++ return element if element < accumulator else accumulator ++ ++ def merge_accumulators(self, accumulators): ++ return min(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator ++ ++ ++class _MaxCombineFn(beam.CombineFn): ++ def create_accumulator(self): ++ return float('-inf') ++ ++ def add_input(self, accumulator, element): ++ return element if element > accumulator else accumulator ++ ++ def merge_accumulators(self, accumulators): ++ return max(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator ++ ++ ++def _get_combiner_for_agg(agg_op): ++ """Map AggOp enum to a Beam CombineFn instance.""" ++ if agg_op == AggOp.SUM: ++ return _SumCombineFn() ++ elif agg_op == AggOp.COUNT: ++ return combiners.CountCombineFn() ++ elif agg_op == AggOp.MIN: ++ return _MinCombineFn() ++ elif agg_op == AggOp.MAX: ++ return _MaxCombineFn() ++ elif agg_op == AggOp.MEAN: ++ return combiners.MeanCombineFn() ++ else: ++ raise ValueError(f"Unknown aggregation operator: {agg_op}") ++ ++ ++class _PreCombineFn(beam.CombineFn): ++ """Stage 1 wrapper: extract_output returns the raw accumulator.""" ++ def __init__(self, combine_fn): ++ self._combine_fn = combine_fn ++ ++ def create_accumulator(self): ++ return self._combine_fn.create_accumulator() ++ ++ def add_input(self, accumulator, element): ++ return self._combine_fn.add_input(accumulator, element) ++ ++ def merge_accumulators(self, accumulators): ++ return self._combine_fn.merge_accumulators(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator # pass raw accumulator, NOT final output ++ ++ ++class _PostCombineFn(beam.CombineFn): ++ """Stage 2 wrapper: add_input merges an accumulator from Stage 1.""" ++ def __init__(self, combine_fn): ++ self._combine_fn = combine_fn ++ ++ def create_accumulator(self): ++ return self._combine_fn.create_accumulator() ++ ++ def add_input(self, accumulator, element): ++ return self._combine_fn.merge_accumulators([accumulator, element]) ++ ++ def merge_accumulators(self, accumulators): ++ return self._combine_fn.merge_accumulators(accumulators) ++ ++ def extract_output(self, accumulator): ++ return self._combine_fn.extract_output(accumulator) ++ ++ ++class _DerivedFieldsFn: ++ """Callable that evaluates derived field expressions on each row dict. ++ ++ Each derived field's ``expression`` is a compiled ``Expr`` callable. ++ This class is passed to ``beam.Map`` and is pickle-safe because ``Expr`` ++ implements ``__reduce__``. ++ """ ++ def __init__(self, derived_fields): ++ self._fields = [(df.name, df.expression) for df in derived_fields] ++ ++ def __call__(self, row): ++ row = dict(row) ++ for name, expr in self._fields: ++ row[name] = expr(row) ++ return row ++ ++ ++class _ApplyMetricExpr(beam.DoFn): ++ """DoFn that evaluates a post-aggregation expression on combined results.""" ++ def __init__(self, measure_combiner, is_keyed): ++ self._measure_combiner = measure_combiner ++ self._is_keyed = is_keyed ++ ++ def process(self, element, window=beam.DoFn.WindowParam): ++ if self._is_keyed: ++ key, agg_dict = element ++ else: ++ agg_dict = element ++ ++ if self._measure_combiner is not None: ++ value = float(self._measure_combiner(agg_dict)) ++ else: ++ value = float(next(iter(agg_dict.values()))) ++ ++ row = beam.Row( ++ value=value, ++ window_start=float(window.start), ++ window_end=float(window.end)) ++ ++ if self._is_keyed: ++ yield (key, row) ++ else: ++ yield row ++ ++ ++class ComputeMetric(beam.PTransform): ++ """Transforms raw row dicts into metric beam.Rows for anomaly detection. ++ ++ Takes a ``PCollection[dict]`` with event-time timestamps and produces ++ either ``PCollection[beam.Row]`` (for global aggregation) or ++ ``PCollection[tuple[key, beam.Row]]`` (for grouped aggregation). ++ ++ The output is directly compatible with ``AnomalyDetection``. ++ ++ Args: ++ metric_spec: A ``MetricSpec`` defining the metric computation. ++ fanout_strategy: Strategy for global (non-keyed) aggregation parallelism. ++ Ignored when group_by is set. Default: SHARDED. ++ fanout: Number of shards for SHARDED or HOTKEY_FANOUT strategies. ++ Ignored for NONE. Default: 400. ++ """ ++ def __init__(self, metric_spec, fanout_strategy=FanoutStrategy.SHARDED, ++ fanout=400): ++ super().__init__() ++ self._spec = metric_spec ++ self._fanout_strategy = fanout_strategy ++ self._fanout = fanout ++ ++ def expand(self, pcoll): ++ spec = self._spec ++ agg = spec.aggregation ++ ++ # Step 1: Apply derived fields ++ if spec.derived_fields: ++ pcoll = pcoll | 'DerivedFields' >> beam.Map( ++ _DerivedFieldsFn(spec.derived_fields)) ++ ++ # Step 2: Apply windowing ++ if agg.window.type == WindowType.FIXED: ++ window_fn = beam_window.FixedWindows(agg.window.size_seconds) ++ elif agg.window.type == WindowType.SLIDING: ++ window_fn = beam_window.SlidingWindows( ++ agg.window.size_seconds, agg.window.period_seconds) ++ else: ++ raise ValueError(f"Unknown window type: {agg.window.type}") ++ ++ windowed = pcoll | 'Window' >> beam.WindowInto(window_fn) ++ ++ # Step 3: Aggregate ++ measures = agg.measures ++ aliases = [m.alias for m in measures] ++ is_keyed = bool(agg.group_by) ++ ++ # Single-measure optimization: skip TupleCombineFn overhead (avoids ++ # tuple creation/unpacking per element on the hot path). ++ if len(measures) == 1: ++ combine_fn = _get_combiner_for_agg(measures[0].agg) ++ _m0 = measures[0] ++ _a0 = aliases[0] ++ ++ def extract_fields(row_dict): ++ return row_dict.get(_m0.field) if _m0.agg != AggOp.COUNT else 1 ++ ++ def to_alias_dict(value): ++ return {_a0: value} ++ else: ++ combine_fn = combiners.TupleCombineFn( ++ *[_get_combiner_for_agg(m.agg) for m in measures]) ++ ++ def extract_fields(row_dict): ++ return tuple( ++ row_dict.get(m.field) if m.agg != AggOp.COUNT else 1 ++ for m in measures) ++ ++ def to_alias_dict(values): ++ return dict(zip(aliases, values)) ++ ++ if is_keyed: ++ group_by_fields = agg.group_by ++ ++ def extract_key_and_fields(row_dict): ++ key = tuple(row_dict.get(f) for f in group_by_fields) ++ return (key, extract_fields(row_dict)) ++ ++ keyed = windowed | 'ExtractKey' >> beam.Map(extract_key_and_fields) ++ aggregated = ( ++ keyed ++ | 'Combine' >> beam.CombinePerKey(combine_fn) ++ | 'ToDict' >> beam.MapTuple(lambda k, v: (k, to_alias_dict(v)))) ++ else: ++ strategy = self._fanout_strategy ++ if strategy == FanoutStrategy.NONE: ++ aggregated = ( ++ windowed ++ | 'ExtractFields' >> beam.Map(extract_fields) ++ | 'Combine' >> beam.CombineGlobally( ++ combine_fn).without_defaults() ++ | 'ToDict' >> beam.Map(to_alias_dict)) ++ elif strategy == FanoutStrategy.HOTKEY_FANOUT: ++ aggregated = ( ++ windowed ++ | 'ExtractFields' >> beam.Map(extract_fields) ++ | 'Combine' >> beam.CombineGlobally( ++ combine_fn).with_fanout(self._fanout).without_defaults() ++ | 'ToDict' >> beam.Map(to_alias_dict)) ++ elif strategy == FanoutStrategy.SHARDED: ++ _num_shards = self._fanout ++ ++ def _shard_fields(row_dict): ++ return (random.randint(0, _num_shards - 1), ++ extract_fields(row_dict)) ++ ++ pre_fn = _PreCombineFn(combine_fn) ++ post_fn = _PostCombineFn(combine_fn) ++ aggregated = ( ++ windowed ++ | 'ShardAndExtract' >> beam.Map(_shard_fields) ++ | 'PartialCombine' >> beam.CombinePerKey(pre_fn) ++ | 'DropShard' >> beam.Values() ++ | 'FinalCombine' >> beam.CombineGlobally( ++ post_fn).without_defaults() ++ | 'ToDict' >> beam.Map(to_alias_dict)) ++ else: ++ raise ValueError(f"Unknown fanout strategy: {strategy}") ++ ++ # Step 4: Apply metric expression and set output type hints ++ metric_dofn = _ApplyMetricExpr(spec.measure_combiner, is_keyed) ++ ++ if is_keyed: ++ # AnomalyDetection checks isinstance(element_type, TupleConstraint) ++ # to detect keyed input. We must annotate the output type. ++ result = aggregated | 'MetricExpr' >> beam.ParDo( ++ metric_dofn).with_output_types(Tuple[Any, beam.Row]) ++ else: ++ result = aggregated | 'MetricExpr' >> beam.ParDo( ++ metric_dofn).with_output_types(beam.Row) ++ ++ return result +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py +new file mode 100644 +index 000000000..d3fa9ea1c +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py +@@ -0,0 +1,1078 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Anomaly monitoring pipeline for BigQuery tables. ++ ++Reads streaming CDC data from BigQuery, computes a configurable windowed ++metric, runs anomaly detection, and publishes anomalies to Pub/Sub. ++ ++Designed to be run as a Dataflow Flex Template or locally with DirectRunner. ++ ++Usage (Flex Template):: ++ ++ gcloud dataflow flex-template run "sales-monitor-$(date +%Y%m%d)" \\ ++ --template-file-gcs-location "gs://bucket/anomaly_monitor.json" \\ ++ --parameters table="project:dataset.table" \\ ++ --parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' \\ ++ --parameters detector_spec='{"type":"ZScore"}' \\ ++ --region us-central1 ++ ++Usage (PrismRunner):: ++ ++ python main.py \\ ++ --table=project:dataset.table \\ ++ --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' \\ ++ --detector_spec='{"type":"ZScore"}' \\ ++ --runner=PrismRunner ++ ++Usage (DataflowRunner):: ++ ++ python main.py \\ ++ --table=project:dataset.table \\ ++ --metric_spec='' \\ ++ --detector_spec='' \\ ++ --runner=DataflowRunner \\ ++ --project=my-project \\ ++ --region=us-central1 \\ ++ --temp_location=gs://bucket/temp \\ ++ --staging_location=gs://bucket/staging \\ ++ --setup_file=./setup.py ++ ++ ++metric_spec JSON Reference ++========================== ++ ++Top-level ``metric_spec`` object:: ++ ++ { ++ "aggregation": { ... }, # required ++ "derived_fields": [ ... ], # optional, pre-aggregation ++ "measure_combiner": { ... } # optional (required if >1 measure) ++ } ++ ++aggregation ++----------- ++:: ++ ++ "aggregation": { ++ "window": { ++ "type": "fixed" | "sliding", ++ "size_seconds": , # window size in seconds ++ "period_seconds": # slide period (required for sliding) ++ }, ++ "group_by": ["field1", "field2"], # optional, omit for global agg ++ "measures": [ ++ {"field": "", "agg": "", "alias": ""}, ++ ... ++ ] ++ } ++ ++Aggregation operators (``agg``): ``SUM``, ``COUNT``, ``MIN``, ``MAX``, ``MEAN``. ++ ++For ``COUNT``, the ``field`` value is ignored — it counts all rows in the ++group. ++ ++Expressions ++----------- ++Both ``measure_combiner.expression`` and ``derived_fields[].expression`` ++are Python expression strings. Bare names are field references, and the ++following syntax is supported: ++ ++- Arithmetic: ``+``, ``-``, ``*``, ``/``, ``//``, ``%``, ``**`` ++- Comparisons: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=`` ++- Boolean logic: ``and``, ``or``, ``not`` ++- Negation: ``-field`` ++- Conditional: ``true_val if condition else false_val`` ++- Functions: ``abs()``, ``min()``, ``max()``, ``round()`` ++- Grouping: parentheses for precedence ++ ++``measure_combiner`` references measure aliases and is validated at ++pipeline construction time. ++ ++derived_fields ++-------------- ++Computed before aggregation. Each entry creates a new column available to ++measures:: ++ ++ "derived_fields": [ ++ {"name": "is_success", "expression": "1 if status == 'success' else 0"} ++ ] ++ ++measure_combiner ++---------------- ++Post-aggregation expression that combines measure aliases into a single ++value. Required when there are multiple measures (e.g., ratio metrics):: ++ ++ "measure_combiner": {"expression": "clicks / impressions"} ++ "measure_combiner": {"expression": "(successes + partial) / total"} ++ ++ ++detector_spec JSON Reference ++============================= ++ ++Top-level ``detector_spec`` object:: ++ ++ {"type": "", "config": { ... }} ++ ++The ``type`` must be a registered ``@specifiable`` detector class name. ++``config`` keys map to that class's ``__init__`` parameters plus inherited ++``AnomalyDetector`` parameters. ++ ++Common AnomalyDetector parameters (all detectors):: ++ ++ "config": { ++ "threshold_criterion": { ... }, # optional, see below ++ "model_id": "" # optional detector ID ++ } ++ ++``features`` is automatically set to ``['value']`` to match ++``ComputeMetric`` output; it does not need to be specified. ++ ++window_size ++----------- ++All detectors maintain an internal sliding window of recent values for their ++statistical trackers (mean, stdev, quantiles, etc.). The default is 1000 ++data points. Use ``window_size`` as a shorthand to override this for all ++internal trackers at once:: ++ ++ {"type": "ZScore", "config": {"window_size": 500}} ++ ++Available detectors ++------------------- ++ ++**ZScore** — ``|value - mean| / stdev`` (default threshold: 3):: ++ ++ {"type": "ZScore"} ++ ++**IQR** — Interquartile Range (default threshold: 1.5):: ++ ++ {"type": "IQR"} ++ ++**RobustZScore** — Modified Z-Score using median/MAD (default threshold: 3.5):: ++ ++ {"type": "RobustZScore"} ++ ++threshold_criterion ++------------------- ++Override the default threshold by nesting a specifiable threshold object. ++ ++**FixedThreshold** — static cutoff (scores >= cutoff are outliers):: ++ ++ "threshold_criterion": { ++ "type": "FixedThreshold", ++ "config": {"cutoff": 10} ++ } ++ ++**QuantileThreshold** — dynamic cutoff at a quantile of observed scores:: ++ ++ "threshold_criterion": { ++ "type": "QuantileThreshold", ++ "config": {"quantile": 0.95} ++ } ++ ++Both accept optional ``normal_label`` (default 0), ``outlier_label`` ++(default 1), and ``missing_label`` (default -2). ++ ++**Threshold** — fixed threshold alert based on a boolean expression. ++No warmup period, no history buffer. Alerts whenever the expression ++evaluates to true:: ++ ++ {"type": "Threshold", "expression": "value >= 0.5"} ++ {"type": "Threshold", "expression": "value > 100 or value < -100"} ++ {"type": "Threshold", "expression": "value <= 0.01"} ++ ++The expression receives the computed metric as ``value`` and supports ++all safe expression operators (see Expressions section above). ++ ++ ++Examples ++-------- ++ ++Simple SUM metric with ZScore:: ++ ++ --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' ++ --detector_spec='{"type":"ZScore"}' ++ ++Grouped ratio metric (CTR) with ZScore:: ++ ++ --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":10},"group_by":["campaign_type","browser_version"],"measures":[{"field":"is_click","agg":"SUM","alias":"clicks"},{"field":"is_click","agg":"COUNT","alias":"impressions"}]},"measure_combiner":{"expression":"clicks / impressions"}}' ++ --detector_spec='{"type":"ZScore"}' ++ ++Derived field + ratio + custom threshold:: ++ ++ --metric_spec='{"derived_fields":[{"name":"is_success","expression":"1 if status == \\'success\\' else 0"}],"aggregation":{"window":{"type":"fixed","size_seconds":10},"group_by":["brand_name","category"],"measures":[{"field":"is_success","agg":"SUM","alias":"successes"},{"field":"is_success","agg":"COUNT","alias":"total"}]},"measure_combiner":{"expression":"successes / total"}}' ++ --detector_spec='{"type":"ZScore","config":{"threshold_criterion":{"type":"FixedThreshold","config":{"cutoff":10}}}}' ++""" ++ ++import datetime ++import json ++import logging ++import re ++import time ++ ++import apache_beam as beam ++from apache_beam.io.gcp.bigquery import WriteToBigQuery ++from apache_beam.io.gcp.pubsub import WriteToPubSub ++from apache_beam.options.pipeline_options import PipelineOptions ++from apache_beam.options.pipeline_options import SetupOptions ++ ++from bqmonitor.metric import ComputeMetric ++from bqmonitor.metric import FanoutStrategy ++from bqmonitor.metric import MetricSpec ++from bqmonitor.safe_eval import Expr ++from apache_beam.ml.anomaly.base import AnomalyPrediction ++from apache_beam.ml.anomaly.base import AnomalyResult ++from apache_beam.ml.anomaly.specifiable import Spec ++from apache_beam.ml.anomaly.specifiable import Specifiable ++from apache_beam.ml.anomaly.transforms import AnomalyDetection ++ ++# Import detectors so they register with @specifiable before from_spec. ++from apache_beam.ml.anomaly.detectors import zscore # noqa: F401 ++from apache_beam.ml.anomaly.detectors import iqr # noqa: F401 ++from apache_beam.ml.anomaly.detectors import robust_zscore # noqa: F401 ++ ++_LOGGER = logging.getLogger(__name__) ++ ++_SUPPORTED_DETECTORS = ('ZScore', 'IQR', 'RobustZScore') ++ ++# Matches project:dataset.table or project.dataset.table ++_TABLE_RE = re.compile( ++ r'^[a-zA-Z0-9][a-zA-Z0-9_-]*[:\.][a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$') ++ ++ ++# --------------------------------------------------------------------------- ++# Helpers ++# --------------------------------------------------------------------------- ++ ++ ++def _validate_topic_path(topic): ++ """Validate that a Pub/Sub topic is a full resource path. ++ ++ Args: ++ topic: Full Pub/Sub topic path, e.g. ++ 'projects/my-project/topics/my-topic'. ++ ++ Returns: ++ The validated topic path. ++ ++ Raises: ++ ValueError: If the topic is not a full resource path. ++ """ ++ if not (topic.startswith('projects/') and '/topics/' in topic): ++ raise ValueError( ++ f"--topic must be a full Pub/Sub resource path " ++ f"(projects//topics/), got: '{topic}'") ++ return topic ++ ++ ++def _unpack_result(element): ++ """Unpack a possibly-keyed AnomalyResult element. ++ ++ Returns: ++ (key, result) where key is None for unkeyed elements. ++ """ ++ if isinstance(element, tuple) and len(element) == 2: ++ return element[0], element[1] ++ return None, element ++ ++ ++def _parse_table_ref(table): ++ """Parse and validate a table reference string. ++ ++ Args: ++ table: Table reference in 'project:dataset.table' or ++ 'project.dataset.table' format. ++ ++ Returns: ++ (project, dataset, table_name) tuple. ++ ++ Raises: ++ ValueError: If the table string doesn't match the expected format. ++ """ ++ if not _TABLE_RE.match(table): ++ raise ValueError( ++ f"Invalid --table format: '{table}'. " ++ f"Expected: project:dataset.table or project.dataset.table") ++ if ':' in table: ++ project, rest = table.split(':', 1) ++ dataset, table_name = rest.split('.', 1) ++ else: ++ project, dataset, table_name = table.split('.', 2) ++ return project, dataset, table_name ++ ++ ++# --------------------------------------------------------------------------- ++# DoFns ++# --------------------------------------------------------------------------- ++ ++ ++class _LogAnomalyResult(beam.DoFn): ++ """Logs each AnomalyResult at WARNING level for visibility in Dataflow.""" ++ def process(self, element): ++ key, result = _unpack_result(element) ++ prediction = result.predictions[0] ++ example = result.example ++ ++ if prediction.label == 1: ++ tag = '!! OUTLIER !!' ++ elif prediction.label == 0: ++ tag = 'NORMAL' ++ else: ++ tag = 'WARMUP' ++ ++ ws = datetime.datetime.fromtimestamp( ++ example.window_start, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ we = datetime.datetime.fromtimestamp( ++ example.window_end, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ window_str = f'{ws}-{we}' ++ ++ if key is not None: ++ _LOGGER.warning( ++ '[%s] window=%s key=%s value=%.2f score=%s label=%s', ++ tag, window_str, key, example.value, prediction.score, ++ prediction.label) ++ else: ++ _LOGGER.warning( ++ '[%s] window=%s value=%.2f score=%s label=%s', ++ tag, window_str, example.value, prediction.score, ++ prediction.label) ++ yield element ++ ++ ++class _ThresholdAlert(beam.DoFn): ++ """Evaluates a threshold expression against metric values. ++ ++ Emits AnomalyResult elements consistent with the statistical detectors, ++ allowing threshold alerts to flow through the same logging and Pub/Sub ++ pipeline. ++ ++ The expression is evaluated with ``value`` bound to the metric value. ++ If it evaluates to truthy, the element is labelled as an outlier (1); ++ otherwise it is labelled normal (0). ++ ++ Example expressions: ``value >= 0.5``, ``value <= 0.01``, ++ ``value > 100 or value < -100``. ++ """ ++ ++ def __init__(self, expression_text): ++ self._expression_text = expression_text ++ self._expr = None ++ ++ def setup(self): ++ self._expr = Expr(self._expression_text) ++ ++ def process(self, element): ++ if isinstance(element, tuple) and len(element) == 2: ++ key, row = element ++ else: ++ key, row = None, element ++ ++ value = row.value ++ is_alert = bool(self._expr({'value': value})) ++ ++ prediction = AnomalyPrediction( ++ model_id=f'Threshold({self._expression_text})', ++ score=None, ++ label=1 if is_alert else 0, ++ threshold=None) ++ ++ result = AnomalyResult(example=row, predictions=[prediction]) ++ ++ if key is not None: ++ yield (key, result) ++ else: ++ yield result ++ ++ ++class _FormatAnomalyAsJson(beam.DoFn): ++ """Converts anomaly results (label == 1) to JSON byte strings for Pub/Sub.""" ++ def process(self, element): ++ key, result = _unpack_result(element) ++ prediction = result.predictions[0] ++ if prediction.label != 1: ++ return ++ ++ example = result.example ++ ws = datetime.datetime.fromtimestamp( ++ example.window_start, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ we = datetime.datetime.fromtimestamp( ++ example.window_end, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ ++ payload = { ++ 'event_description': ( ++ f'Anomaly detected value={example.value}' ++ f' score={prediction.score}' ++ f' in window={ws}-{we}'), ++ 'agent_id': prediction.model_id, ++ } ++ if key is not None: ++ payload['key'] = str(key) ++ ++ yield json.dumps(payload).encode('utf-8') ++ ++ ++_SINK_SCHEMA = { ++ 'fields': [ ++ {'name': 'window_start', 'type': 'TIMESTAMP', 'mode': 'REQUIRED'}, ++ {'name': 'window_end', 'type': 'TIMESTAMP', 'mode': 'REQUIRED'}, ++ {'name': 'value', 'type': 'FLOAT64', 'mode': 'REQUIRED'}, ++ {'name': 'score', 'type': 'FLOAT64', 'mode': 'NULLABLE'}, ++ {'name': 'label', 'type': 'INT64', 'mode': 'REQUIRED'}, ++ {'name': 'key', 'type': 'STRING', 'mode': 'NULLABLE'}, ++ ] ++} ++ ++ ++class _FormatResultForBQ(beam.DoFn): ++ """Converts all AnomalyResult elements to BQ row dicts.""" ++ def process(self, element): ++ key, result = _unpack_result(element) ++ prediction = result.predictions[0] ++ example = result.example ++ ++ row = { ++ 'window_start': datetime.datetime.fromtimestamp( ++ example.window_start, tz=datetime.timezone.utc).isoformat(), ++ 'window_end': datetime.datetime.fromtimestamp( ++ example.window_end, tz=datetime.timezone.utc).isoformat(), ++ 'value': float(example.value), ++ 'score': float(prediction.score) if prediction.score is not None ++ else None, ++ 'label': int(prediction.label), ++ } ++ if key is not None: ++ row['key'] = str(key) ++ ++ yield row ++ ++ ++# --------------------------------------------------------------------------- ++# Pipeline options ++# --------------------------------------------------------------------------- ++ ++ ++class AnomalyMonitorOptions(PipelineOptions): ++ """Pipeline options for the anomaly monitor.""" ++ @classmethod ++ def _add_argparse_args(cls, parser): ++ parser.add_argument( ++ '--table', ++ default=None, ++ help='BigQuery table to monitor. ' ++ 'Format: project:dataset.table') ++ parser.add_argument( ++ '--metric_spec', ++ default=None, ++ help='JSON string defining the metric computation. ' ++ 'See MetricSpec.from_dict() for schema.') ++ parser.add_argument( ++ '--detector_spec', ++ default=None, ++ help='JSON string defining the anomaly detector. ' ++ 'Format: {"type":"ZScore"} or ' ++ '{"type":"ZScore","config":{"threshold_criterion":{...}}}') ++ parser.add_argument( ++ '--poll_interval_sec', ++ type=int, ++ default=60, ++ help='Seconds between BigQuery CDC polls. Default 60.') ++ parser.add_argument( ++ '--change_function', ++ default='APPENDS', ++ choices=['APPENDS', 'CHANGES'], ++ help='BigQuery change function to use. Default APPENDS.') ++ parser.add_argument( ++ '--buffer_sec', ++ type=float, ++ default=15.0, ++ help='Safety buffer behind now() in seconds. Default 15.') ++ parser.add_argument( ++ '--start_offset_sec', ++ type=float, ++ default=60.0, ++ help='Start reading from this many seconds ago. Default 60.') ++ parser.add_argument( ++ '--duration_sec', ++ type=float, ++ default=0.0, ++ help='How long to run in seconds. 0 means run forever. Default 0.') ++ parser.add_argument( ++ '--temp_dataset', ++ default=None, ++ help='BigQuery dataset for temp tables. If unset, auto-created.') ++ parser.add_argument( ++ '--topic', ++ default=None, ++ help='Pub/Sub topic for anomaly results. ' ++ 'Full path: projects//topics/.') ++ parser.add_argument( ++ '--log_all_results', ++ default='false', ++ help='Log all anomaly detection results (normal, outlier, warmup) ' ++ 'at WARNING level. Default: false.') ++ parser.add_argument( ++ '--sink_table', ++ default=None, ++ help='BigQuery table to write all anomaly detection results to. ' ++ 'Format: project:dataset.table. If unset, results are not written ' ++ 'to BigQuery.') ++ parser.add_argument( ++ '--decompress_shards', ++ type=int, ++ default=400, ++ help='Number of shards for CDC Arrow batch decompression fan-out. ' ++ 'Spreads decompression CPU across workers. ' ++ '0 disables fan-out (decode inline). Default: 400.') ++ parser.add_argument( ++ '--fanout_strategy', ++ default='sharded', ++ choices=['sharded', 'hotkey_fanout', 'none'], ++ help='Parallelism strategy for global (non-keyed) metric ' ++ 'aggregation. Ignored when group_by is set. Default: sharded.') ++ parser.add_argument( ++ '--fanout', ++ type=int, ++ default=400, ++ help='Number of shards for sharded or hotkey_fanout strategies. ' ++ 'Ignored for none. Default: 400.') ++ ++ ++# --------------------------------------------------------------------------- ++# Spec parsing ++# --------------------------------------------------------------------------- ++ ++ ++def _parse_metric_spec(json_str): ++ """Parse a MetricSpec from a JSON string. ++ ++ Raises: ++ ValueError: If the JSON is malformed or the spec is invalid. ++ """ ++ try: ++ d = json.loads(json_str) ++ except json.JSONDecodeError as e: ++ raise ValueError( ++ f"Invalid JSON in --metric_spec: {e}. " ++ f"Value: {json_str[:200]}") from e ++ try: ++ return MetricSpec.from_dict(d) ++ except (ValueError, TypeError, KeyError) as e: ++ raise ValueError(f"Invalid --metric_spec: {e}") from e ++ ++ ++def _dict_to_spec(d): ++ """Recursively convert nested dicts with ``type`` keys into Spec objects. ++ ++ ``json.loads`` produces plain dicts, but ``Specifiable.from_spec`` expects ++ ``Spec`` objects for nested specifiables (e.g. ``threshold_criterion`` ++ inside a detector config). Without this conversion the nested dict passes ++ through ``_specifiable_from_spec_helper`` unchanged and the detector ++ receives a raw dict instead of the expected ``ThresholdFn`` instance. ++ """ ++ if isinstance(d, dict) and 'type' in d: ++ config = d.get('config', {}) ++ if config: ++ config = {k: _dict_to_spec(v) for k, v in config.items()} ++ return Spec(type=d['type'], config=config) ++ if isinstance(d, list): ++ return [_dict_to_spec(item) for item in d] ++ return d ++ ++ ++def _expand_window_size(d): ++ """Expand ``window_size`` shorthand into detector-specific tracker configs. ++ ++ Instead of constructing deeply nested tracker specs, users can write:: ++ ++ {"type": "ZScore", "config": {"window_size": 500}} ++ ++ This expands into the full nested tracker configuration that each detector ++ type expects. If the user already set explicit tracker configs, those take ++ precedence (``setdefault`` semantics). ++ ++ Raises: ++ ValueError: If window_size is not a positive integer. ++ """ ++ config = d.get('config', {}) ++ ws = config.pop('window_size', None) ++ if ws is None: ++ return ++ ++ if not isinstance(ws, int) or ws <= 0: ++ raise ValueError( ++ f"window_size must be a positive integer, got {ws!r}") ++ ++ detector_type = d['type'] ++ ++ if detector_type == 'ZScore': ++ config.setdefault( ++ 'sub_stat_tracker', ++ {'type': 'IncSlidingMeanTracker', 'config': {'window_size': ws}}) ++ config.setdefault( ++ 'stdev_tracker', ++ {'type': 'IncSlidingStdevTracker', 'config': {'window_size': ws}}) ++ elif detector_type == 'IQR': ++ config.setdefault( ++ 'q1_tracker', ++ { ++ 'type': 'BufferedSlidingQuantileTracker', ++ 'config': {'window_size': ws, 'q': 0.25} ++ }) ++ # q3_tracker auto-derives from q1_tracker in IQR.__init__ ++ elif detector_type == 'RobustZScore': ++ _median_tracker_spec = { ++ 'type': 'MedianTracker', ++ 'config': { ++ 'quantile_tracker': { ++ 'type': 'BufferedSlidingQuantileTracker', ++ 'config': {'window_size': ws, 'q': 0.5} ++ } ++ } ++ } ++ config.setdefault( ++ 'mad_tracker', ++ { ++ 'type': 'MadTracker', ++ 'config': { ++ 'median_tracker': _median_tracker_spec, ++ 'diff_median_tracker': { ++ 'type': 'MedianTracker', ++ 'config': { ++ 'quantile_tracker': { ++ 'type': 'BufferedSlidingQuantileTracker', ++ 'config': {'window_size': ws, 'q': 0.5} ++ } ++ } ++ } ++ } ++ }) ++ ++ ++def _parse_detector_spec(json_str): ++ """Parse an anomaly detector from a JSON Spec string. ++ ++ The JSON should have the form:: ++ ++ {"type": "ZScore"} ++ ++ Nested specifiable objects (e.g. ``threshold_criterion``) are supported:: ++ ++ {"type": "ZScore", "config": { ++ "threshold_criterion": {"type": "FixedThreshold", "config": {"cutoff": 10}} ++ }} ++ ++ A ``window_size`` shorthand sets the history buffer for all internal ++ trackers:: ++ ++ {"type": "ZScore", "config": {"window_size": 500}} ++ ++ **Threshold** — a simple fixed-threshold alerter that evaluates a boolean ++ expression against the metric value. No warmup period, no history:: ++ ++ {"type": "Threshold", "expression": "value >= 0.5"} ++ {"type": "Threshold", "expression": "value > 100 or value < -100"} ++ ++ The expression may use ``value`` (the computed metric) and all safe ++ expression operators (see Expressions section above). ++ ++ For statistical detectors, the ``type`` field must match a registered ++ @specifiable detector class (e.g. ZScore, IQR, RobustZScore). ++ ++ ``features`` is automatically set to ``['value']`` to match the output of ++ ``ComputeMetric``. Any user-supplied ``features`` is overwritten. ++ ++ Returns: ++ For statistical detectors: an instantiated AnomalyDetector. ++ For Threshold: a ``_ThresholdAlert`` DoFn instance. ++ ++ Raises: ++ ValueError: If the JSON is malformed, detector type is unknown, or ++ the spec is otherwise invalid. ++ """ ++ try: ++ d = json.loads(json_str) ++ except json.JSONDecodeError as e: ++ raise ValueError( ++ f"Invalid JSON in --detector_spec: {e}. " ++ f"Value: {json_str[:200]}") from e ++ ++ if not isinstance(d, dict) or 'type' not in d: ++ raise ValueError( ++ "detector_spec must be a JSON object with a 'type' field. " ++ f"Example: {{\"type\":\"ZScore\"}}. Got: {json_str[:200]}") ++ ++ detector_type = d['type'] ++ ++ if detector_type == 'Threshold': ++ expr_text = d.get('expression') ++ if not expr_text: ++ raise ValueError( ++ "Threshold detector requires an 'expression' field. " ++ "Example: {\"type\":\"Threshold\",\"expression\":\"value >= 0.5\"}") ++ # Validate the expression at parse time. ++ try: ++ expr = Expr(expr_text) ++ except (ValueError, SyntaxError) as e: ++ raise ValueError( ++ f"Invalid threshold expression: {e}") from e ++ if 'value' not in expr.field_refs(): ++ _LOGGER.warning( ++ "Threshold expression '%s' does not reference 'value'. " ++ "It will receive the computed metric value as 'value'.", expr_text) ++ return _ThresholdAlert(expr_text) ++ ++ if detector_type not in _SUPPORTED_DETECTORS: ++ raise ValueError( ++ f"Unknown detector type '{detector_type}'. " ++ f"Supported detectors: {', '.join(_SUPPORTED_DETECTORS)}, Threshold") ++ ++ d.setdefault('config', {}) ++ d['config']['features'] = ['value'] ++ _expand_window_size(d) ++ spec = _dict_to_spec(d) ++ try: ++ return Specifiable.from_spec(spec, _run_init=True) ++ except (ValueError, TypeError) as e: ++ raise ValueError( ++ f"Failed to construct {detector_type} detector: {e}") from e ++ ++ ++# --------------------------------------------------------------------------- ++# Preflight checks ++# --------------------------------------------------------------------------- ++ ++ ++def _preflight_checks(options, metric_spec): ++ """Validate GCP resources are accessible before building the pipeline. ++ ++ Checks: ++ - BigQuery source table exists and is readable. ++ - Required metric columns exist in the source table (dry-run query). ++ - BigQuery temp dataset is writable (if specified) or datasets.create ++ permission exists (dry-run only — does not actually create). ++ - Pub/Sub topic exists. ++ ++ Logs warnings and continues if a check cannot be performed (e.g. missing ++ client library). Raises ValueError on definite failures. ++ """ ++ project, dataset, table_name = _parse_table_ref(options.table) ++ topic_path = _validate_topic_path(options.topic) ++ ++ required_columns = sorted(metric_spec.required_source_columns()) ++ _check_bq_source_table(project, dataset, table_name, options, ++ required_columns) ++ _check_bq_temp_dataset(project, options) ++ _check_pubsub_topic(topic_path) ++ ++ ++def _check_bq_source_table(project, dataset, table_name, options, ++ required_columns): ++ """Verify the source BigQuery table exists and required columns are present. ++ ++ Runs a dry-run CDC query selecting the columns referenced by the metric ++ spec. This validates table access, CDC function access, and column ++ existence in a single round-trip. ++ """ ++ try: ++ from apache_beam.io.gcp import bigquery_tools ++ from apache_beam.io.gcp.internal.clients import bigquery ++ except ImportError: ++ _LOGGER.warning( ++ '[Preflight] Skipping BQ table check: ' ++ 'BigQuery client libraries not available') ++ return ++ ++ try: ++ bq = bigquery_tools.BigQueryWrapper() ++ bq.get_table(project, dataset, table_name) ++ _LOGGER.info( ++ '[Preflight] Source table %s:%s.%s is accessible', ++ project, dataset, table_name) ++ except Exception as e: ++ raise ValueError( ++ f"Cannot access BigQuery table '{project}:{dataset}.{table_name}'. " ++ f"Verify it exists and the service account has " ++ f"bigquery.tables.get and bigquery.tables.getData permissions. " ++ f"Error: {e}") from e ++ ++ # Dry-run a CDC query selecting the metric's required columns. ++ # This validates CDC function access and column existence in one step. ++ select_clause = ', '.join(required_columns) if required_columns else '1' ++ try: ++ sql = ( ++ f"SELECT {select_clause} FROM {options.change_function}" ++ f"(TABLE `{project}.{dataset}.{table_name}`, " ++ f"NULL, NULL) LIMIT 0") ++ _LOGGER.info('[Preflight] Dry-run query: %s', sql) ++ request = bigquery.BigqueryJobsInsertRequest( ++ projectId=project, ++ job=bigquery.Job( ++ configuration=bigquery.JobConfiguration( ++ query=bigquery.JobConfigurationQuery( ++ query=sql, ++ useLegacySql=False), ++ dryRun=True))) ++ bq.client.jobs.Insert(request) ++ _LOGGER.info( ++ '[Preflight] %s() access and columns %s verified for %s:%s.%s', ++ options.change_function, required_columns, ++ project, dataset, table_name) ++ except Exception as e: ++ raise ValueError( ++ f"Cannot execute {options.change_function}() on " ++ f"'{project}:{dataset}.{table_name}' " ++ f"with columns {required_columns}. " ++ f"Verify the table has change history enabled, the columns exist, " ++ f"and the service account has bigquery.jobs.create permission. " ++ f"Error: {e}") from e ++ ++ ++def _check_bq_temp_dataset(project, options): ++ """Verify access to the temp dataset (if specified), or check that ++ datasets.create permission exists for auto-creation.""" ++ try: ++ from apache_beam.io.gcp import bigquery_tools ++ from apache_beam.io.gcp.internal.clients import bigquery ++ from apitools.base.py.exceptions import HttpError ++ except ImportError: ++ _LOGGER.warning( ++ '[Preflight] Skipping BQ temp dataset check: ' ++ 'BigQuery client libraries not available') ++ return ++ ++ if options.temp_dataset: ++ try: ++ bq = bigquery_tools.BigQueryWrapper() ++ bq.client.datasets.Get( ++ bigquery.BigqueryDatasetsGetRequest( ++ projectId=project, datasetId=options.temp_dataset)) ++ _LOGGER.info( ++ '[Preflight] Temp dataset %s:%s exists', ++ project, options.temp_dataset) ++ except HttpError as e: ++ if e.status_code == 404: ++ raise ValueError( ++ f"Temp dataset '{project}:{options.temp_dataset}' not found. " ++ f"Create it or omit --temp_dataset for auto-creation.") from e ++ elif e.status_code == 403: ++ raise ValueError( ++ f"No access to temp dataset '{project}:{options.temp_dataset}'. " ++ f"Verify the service account has " ++ f"bigquery.datasets.get permission.") from e ++ raise ++ ++ # Verify we can write to the temp dataset by doing a dry-run query ++ # with a destination table in it. ++ try: ++ temp_table_ref = bigquery.TableReference( ++ projectId=project, ++ datasetId=options.temp_dataset, ++ tableId='beam_ch_preflight_check') ++ request = bigquery.BigqueryJobsInsertRequest( ++ projectId=project, ++ job=bigquery.Job( ++ configuration=bigquery.JobConfiguration( ++ query=bigquery.JobConfigurationQuery( ++ query='SELECT 1', ++ useLegacySql=False, ++ destinationTable=temp_table_ref, ++ writeDisposition='WRITE_TRUNCATE'), ++ dryRun=True))) ++ bq.client.jobs.Insert(request) ++ _LOGGER.info( ++ '[Preflight] Write access to temp dataset %s:%s verified', ++ project, options.temp_dataset) ++ except Exception as e: ++ raise ValueError( ++ f"Cannot write to temp dataset '{project}:{options.temp_dataset}'. " ++ f"Verify the service account has bigquery.tables.create and " ++ f"bigquery.tables.updateData permissions on this dataset. " ++ f"Error: {e}") from e ++ else: ++ _LOGGER.info( ++ '[Preflight] No --temp_dataset specified; ' ++ 'will auto-create at runtime (requires bigquery.datasets.create)') ++ ++ ++def _check_pubsub_topic(topic_path): ++ """Verify the Pub/Sub topic exists.""" ++ try: ++ from google.cloud import pubsub_v1 ++ from google.api_core.exceptions import NotFound, PermissionDenied ++ except ImportError: ++ _LOGGER.warning( ++ '[Preflight] Skipping Pub/Sub check: ' ++ 'google-cloud-pubsub not available') ++ return ++ ++ try: ++ publisher = pubsub_v1.PublisherClient() ++ publisher.get_topic(topic=topic_path) ++ _LOGGER.info('[Preflight] Pub/Sub topic %s is accessible', topic_path) ++ except NotFound: ++ raise ValueError( ++ f"Pub/Sub topic '{topic_path}' not found. " ++ f"Create it with: gcloud pubsub topics create {topic_path}") ++ except PermissionDenied as e: ++ raise ValueError( ++ f"No permission to access Pub/Sub topic '{topic_path}'. " ++ f"Verify the service account has pubsub.topics.get and " ++ f"pubsub.topics.publish permissions. Error: {e}") from e ++ except Exception as e: ++ _LOGGER.warning( ++ '[Preflight] Could not verify Pub/Sub topic %s: %s', ++ topic_path, e) ++ ++ ++# --------------------------------------------------------------------------- ++# Pipeline construction ++# --------------------------------------------------------------------------- ++ ++ ++def build_pipeline(pipeline, options, metric_spec, detector): ++ """Construct the anomaly monitoring pipeline. ++ ++ Args: ++ pipeline: A beam.Pipeline instance. ++ options: AnomalyMonitorOptions with table, poll_interval_sec, etc. ++ metric_spec: Parsed MetricSpec instance. ++ detector: Parsed anomaly detector instance. ++ ++ Returns: ++ The final PCollection (for testing). ++ """ ++ from bqmonitor.cdc import ReadBigQueryChangeHistory ++ ++ start_time = time.time() - options.start_offset_sec ++ stop_time = ( ++ time.time() + options.duration_sec if options.duration_sec > 0 else None) ++ ++ _LOGGER.info('Anomaly Monitor Pipeline') ++ _LOGGER.info(' Table: %s', options.table) ++ _LOGGER.info(' Detector: %s', type(detector).__name__) ++ _LOGGER.info(' Poll interval: %d sec', options.poll_interval_sec) ++ _LOGGER.info(' Change function: %s', options.change_function) ++ ++ columns = sorted(metric_spec.required_source_columns()) ++ _LOGGER.info(' Columns: %s', columns) ++ ++ # Auto-rename pseudo-columns if they conflict with user column names. ++ change_type_col = 'change_type' ++ change_ts_col = 'change_timestamp' ++ col_set = set(columns) ++ if change_type_col in col_set: ++ change_type_col = '_bqm_change_type' ++ _LOGGER.info( ++ ' Renamed pseudo-column change_type -> %s to avoid conflict', ++ change_type_col) ++ if change_ts_col in col_set: ++ change_ts_col = '_bqm_change_timestamp' ++ _LOGGER.info( ++ ' Renamed pseudo-column change_timestamp -> %s to avoid conflict', ++ change_ts_col) ++ ++ cdc_kwargs = dict( ++ table=options.table, ++ poll_interval_sec=options.poll_interval_sec, ++ start_time=start_time, ++ change_function=options.change_function, ++ buffer_sec=options.buffer_sec, ++ columns=columns, ++ change_type_column=change_type_col, ++ change_timestamp_column=change_ts_col, ++ decompress_shards=( ++ options.decompress_shards if options.decompress_shards > 0 ++ else None)) ++ if stop_time is not None: ++ cdc_kwargs['stop_time'] = stop_time ++ if options.temp_dataset: ++ cdc_kwargs['temp_dataset'] = options.temp_dataset ++ ++ rows = pipeline | 'ReadCDC' >> ReadBigQueryChangeHistory(**cdc_kwargs) ++ fanout_strategy = FanoutStrategy(options.fanout_strategy) ++ metrics = rows | 'ComputeMetric' >> ComputeMetric( ++ metric_spec, fanout_strategy=fanout_strategy, fanout=options.fanout) ++ ++ # Rewindow into GlobalWindows so the anomaly detector sees the full ++ # stream of window results as a time series, not isolated per-window. ++ from apache_beam.transforms.window import GlobalWindows ++ global_metrics = metrics | 'Rewindow' >> beam.WindowInto(GlobalWindows()) ++ ++ if isinstance(detector, _ThresholdAlert): ++ anomalies = global_metrics | 'DetectAnomalies' >> beam.ParDo(detector) ++ else: ++ anomalies = global_metrics | 'DetectAnomalies' >> AnomalyDetection(detector) ++ ++ if options.log_all_results.lower() == 'true': ++ _ = anomalies | 'LogResults' >> beam.ParDo(_LogAnomalyResult()) ++ ++ # Publish anomalies (label == 1) to Pub/Sub. ++ topic_path = _validate_topic_path(options.topic) ++ ++ _ = ( ++ anomalies ++ | 'FormatAnomalies' >> beam.ParDo(_FormatAnomalyAsJson()) ++ | 'WriteToPubSub' >> WriteToPubSub(topic=topic_path)) ++ ++ # Write all results to a BigQuery sink table (if configured). ++ if options.sink_table: ++ sink_table = options.sink_table.replace(':', '.') ++ _ = ( ++ anomalies ++ | 'FormatForBQ' >> beam.ParDo(_FormatResultForBQ()) ++ | 'WriteSink' >> WriteToBigQuery( ++ table=sink_table, ++ method='STREAMING_INSERTS', ++ schema=_SINK_SCHEMA, ++ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, ++ write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)) ++ ++ return anomalies ++ ++ ++def run(argv=None): ++ """Main entry point.""" ++ options = PipelineOptions(argv) ++ monitor_options = options.view_as(AnomalyMonitorOptions) ++ ++ # Validate required options. ++ for required_opt in ('table', 'metric_spec', 'detector_spec', 'topic'): ++ if getattr(monitor_options, required_opt) is None: ++ raise ValueError(f'--{required_opt} is required') ++ ++ # Validate table format. ++ _parse_table_ref(monitor_options.table) ++ ++ # Parse specs early so errors surface before pipeline construction. ++ metric_spec = _parse_metric_spec(monitor_options.metric_spec) ++ detector = _parse_detector_spec(monitor_options.detector_spec) ++ ++ # Check GCP resources are accessible. ++ _preflight_checks(monitor_options, metric_spec) ++ ++ options.view_as(SetupOptions).save_main_session = True ++ ++ with beam.Pipeline(options=options) as p: ++ build_pipeline(p, monitor_options, metric_spec, detector) ++ ++ ++if __name__ == '__main__': ++ logging.getLogger().setLevel(logging.INFO) ++ run() +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py +new file mode 100644 +index 000000000..3cc9e00d2 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py +@@ -0,0 +1,192 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Safe expression evaluator for metric computation. ++ ++Parses a Python expression string, validates it uses only allowed ++constructs, compiles it, and produces a callable that evaluates the ++expression against a field context dict. ++ ++Example usage:: ++ ++ from bqmonitor.safe_eval import Expr ++ ++ expr = Expr.from_string("clicks / impressions") ++ expr({'clicks': 50, 'impressions': 1000}) # 0.05 ++ ++ expr = Expr.from_string("1 if status == 'success' else 0") ++ expr({'status': 'success'}) # 1 ++ ++Allowed constructs: field names (bare names), literals (int, float, str), ++arithmetic (``+, -, *, /, //, %, **``), comparisons (``==, !=, <, <=, >, >=``), ++boolean logic (``and, or, not``), unary negation, ``if/else``, and ++safe builtins (``abs, min, max, round``). ++""" ++ ++import ast ++ ++# --- AST whitelist --- ++ ++_ALLOWED_BINOPS = ( ++ ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow) ++ ++_ALLOWED_CMPOPS = (ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE) ++ ++_SAFE_BUILTINS = {"abs": abs, "min": min, "max": max, "round": round} ++ ++ ++def _validate_ast(node): ++ """Recursively validate that an AST node uses only allowed constructs.""" ++ if isinstance(node, ast.Name): ++ return ++ ++ if isinstance(node, ast.Constant): ++ if not isinstance(node.value, (int, float, str)): ++ raise ValueError( ++ f"Unsupported literal type: {type(node.value).__name__}. " ++ f"Only int, float, and str literals are supported.") ++ return ++ ++ if isinstance(node, ast.UnaryOp): ++ if not isinstance(node.op, (ast.USub, ast.Not)): ++ raise ValueError( ++ f"Unsupported unary operator: {type(node.op).__name__}. " ++ f"Only negation (-) and not are supported.") ++ _validate_ast(node.operand) ++ return ++ ++ if isinstance(node, ast.BinOp): ++ if not isinstance(node.op, _ALLOWED_BINOPS): ++ raise ValueError( ++ f"Unsupported binary operator: {type(node.op).__name__}. " ++ f"Supported: +, -, *, /, //, %, **") ++ _validate_ast(node.left) ++ _validate_ast(node.right) ++ return ++ ++ if isinstance(node, ast.BoolOp): ++ # and / or ++ for value in node.values: ++ _validate_ast(value) ++ return ++ ++ if isinstance(node, ast.Compare): ++ if len(node.ops) != 1 or len(node.comparators) != 1: ++ raise ValueError( ++ "Chained comparisons not supported (e.g., a < b < c). " ++ "Use (a < b) and separate expressions instead.") ++ if not isinstance(node.ops[0], _ALLOWED_CMPOPS): ++ raise ValueError( ++ f"Unsupported comparison: {type(node.ops[0]).__name__}. " ++ f"Supported: ==, !=, <, <=, >, >=") ++ _validate_ast(node.left) ++ _validate_ast(node.comparators[0]) ++ return ++ ++ if isinstance(node, ast.IfExp): ++ _validate_ast(node.test) ++ _validate_ast(node.body) ++ _validate_ast(node.orelse) ++ return ++ ++ if isinstance(node, ast.Call): ++ if not (isinstance(node.func, ast.Name) ++ and node.func.id in _SAFE_BUILTINS): ++ name = node.func.id if isinstance(node.func, ast.Name) else ast.dump( ++ node.func) ++ raise ValueError( ++ f"Unsupported function: {name}. " ++ f"Supported: {', '.join(sorted(_SAFE_BUILTINS))}.") ++ if node.keywords: ++ raise ValueError("Keyword arguments not supported in function calls.") ++ for arg in node.args: ++ _validate_ast(arg) ++ return ++ ++ raise ValueError( ++ f"Unsupported expression: {ast.dump(node)}. " ++ f"Only field names, literals, arithmetic (+,-,*,/,//,%,**), " ++ f"comparisons (==,!=,<,<=,>,>=), boolean logic (and, or, not), " ++ f"if/else, and functions ({', '.join(sorted(_SAFE_BUILTINS))}) " ++ f"are supported.") ++ ++ ++def _collect_field_refs(node): ++ """Collect all field names referenced in an AST (excluding builtins).""" ++ return frozenset( ++ child.id for child in ast.walk(node) ++ if isinstance(child, ast.Name) and child.id not in _SAFE_BUILTINS) ++ ++ ++class Expr: ++ """A validated, compiled expression callable. ++ ++ Parses a Python expression string, validates it uses only safe ++ constructs, and compiles it into a callable. The compiled expression ++ is evaluated with restricted builtins (no access to ``import``, ++ ``open``, ``exec``, etc.). ++ ++ Args: ++ text: A Python expression string. ++ ++ Raises: ++ ValueError: If the expression uses unsupported Python constructs. ++ SyntaxError: If the string is not valid Python syntax. ++ """ ++ def __init__(self, text): ++ self._text = text ++ tree = ast.parse(text, mode='eval') ++ _validate_ast(tree.body) ++ self._code = compile(tree, '', 'eval') ++ self._refs = _collect_field_refs(tree.body) ++ ++ def __call__(self, context): ++ """Evaluate the expression against a dict of field values.""" ++ env = dict(_SAFE_BUILTINS) ++ env.update(context) ++ return eval(self._code, {"__builtins__": {}}, env) ++ ++ def field_refs(self): ++ """Return the set of field names referenced by this expression.""" ++ return set(self._refs) ++ ++ @staticmethod ++ def from_string(text): ++ """Parse a Python expression string into a compiled Expr callable. ++ ++ Examples:: ++ ++ Expr.from_string("clicks / impressions") ++ Expr.from_string("1 if status == 'success' else 0") ++ Expr.from_string("(a + b) / total") ++ """ ++ return Expr(text) ++ ++ def __str__(self): ++ return self._text ++ ++ def __repr__(self): ++ return f"Expr({self._text!r})" ++ ++ def __eq__(self, other): ++ return isinstance(other, Expr) and self._text == other._text ++ ++ def __hash__(self): ++ return hash(self._text) ++ ++ def __reduce__(self): ++ """Pickle support: store text, recompile on unpickle.""" ++ return (Expr, (self._text, )) +diff --git a/python/src/test/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetectionIT.java b/python/src/test/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetectionIT.java +new file mode 100644 +index 000000000..dc6200ce1 +--- /dev/null ++++ b/python/src/test/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetectionIT.java +@@ -0,0 +1,584 @@ ++/* ++ * Copyright (C) 2026 Google LLC ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); you may not ++ * use this file except in compliance with the License. You may obtain a copy of ++ * the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++ * License for the specific language governing permissions and limitations under ++ * the License. ++ */ ++package com.google.cloud.teleport.templates.python; ++ ++import static com.google.common.truth.Truth.assertThat; ++import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; ++import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; ++ ++import com.google.cloud.bigquery.Field; ++import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; ++import com.google.cloud.bigquery.Schema; ++import com.google.cloud.bigquery.StandardSQLTypeName; ++import com.google.cloud.bigquery.TableResult; ++import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; ++import com.google.cloud.teleport.metadata.TemplateIntegrationTest; ++import com.google.common.collect.ImmutableMap; ++import com.google.pubsub.v1.ReceivedMessage; ++import com.google.pubsub.v1.SubscriptionName; ++import com.google.pubsub.v1.TopicName; ++import java.io.IOException; ++import java.time.Instant; ++import java.util.ArrayList; ++import java.util.HashSet; ++import java.util.List; ++import java.util.Map; ++import java.util.Random; ++import java.util.Set; ++import java.util.concurrent.TimeUnit; ++import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; ++import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; ++import org.apache.beam.it.common.PipelineOperator.Result; ++import org.apache.beam.it.common.utils.ResourceManagerUtils; ++import org.apache.beam.it.gcp.TemplateTestBase; ++import org.apache.beam.it.gcp.bigquery.BigQueryResourceManager; ++import org.apache.beam.it.gcp.bigquery.matchers.BigQueryAsserts; ++import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; ++import org.apache.beam.it.gcp.pubsub.conditions.PubsubMessagesCheck; ++import org.json.JSONObject; ++import org.junit.After; ++import org.junit.Before; ++import org.junit.Test; ++import org.junit.experimental.categories.Category; ++import org.junit.runner.RunWith; ++import org.junit.runners.JUnit4; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; ++ ++/** ++ * Integration test for {@link BigQueryAnomalyDetection}. ++ * ++ *

Inserts normal baseline data into a BQ table, then injects an anomalous spike, and verifies ++ * that the pipeline detects the anomaly and publishes it to Pub/Sub. ++ */ ++@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) ++@TemplateIntegrationTest(BigQueryAnomalyDetection.class) ++@RunWith(JUnit4.class) ++public final class BigQueryAnomalyDetectionIT extends TemplateTestBase { ++ ++ private static final Logger LOG = LoggerFactory.getLogger(BigQueryAnomalyDetectionIT.class); ++ ++ private static final String TABLE_NAME = "anomaly_test"; ++ private static final String SINK_TABLE_NAME = "anomaly_results"; ++ ++ // Window size used in all metric specs (seconds). ++ private static final int WINDOW_SIZE_SEC = 1; ++ ++ // Baseline: insert 100 rows every second for ~6 minutes (360 batches). ++ // Each batch lands in one 1-second window, giving the detector many data ++ // points to warm up before the anomaly spike. ++ private static final int BASELINE_BATCHES = 360; ++ private static final int ROWS_PER_BATCH = 100; ++ private static final long BATCH_INTERVAL_MS = 1000; ++ private static final double NORMAL_AMOUNT = 10.0; ++ ++ // Anomaly: 100 rows with amount=10000.0 → window MEAN ≈ 10000.0 (1000x spike). ++ private static final double ANOMALY_AMOUNT = 10000.0; ++ ++ private BigQueryResourceManager bigQueryResourceManager; ++ private PubsubResourceManager pubsubResourceManager; ++ ++ @Before ++ public void setUp() throws IOException { ++ bigQueryResourceManager = ++ BigQueryResourceManager.builder(testName, PROJECT, credentials).build(); ++ pubsubResourceManager = ++ PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); ++ } ++ ++ @After ++ public void cleanUp() { ++ ResourceManagerUtils.cleanResources(bigQueryResourceManager, pubsubResourceManager); ++ } ++ ++ @Test ++ public void testDetectsAnomalyAndPublishesToPubSub() throws IOException, InterruptedException { ++ testSimpleSumMetric(); ++ } ++ ++ @Test ++ public void testGroupedRatioMetric() throws IOException, InterruptedException { ++ testGroupedRatioMetricImpl(); ++ } ++ ++ @Test ++ public void testThresholdDetector() throws IOException, InterruptedException { ++ testThresholdDetectorImpl(); ++ } ++ ++ // ------------------------------------------------------------------------- ++ // Test implementations ++ // ------------------------------------------------------------------------- ++ ++ private void testSimpleSumMetric() throws IOException, InterruptedException { ++ // --- Arrange --- ++ ++ // Create BQ table. APPENDS() does not require change tracking. ++ Schema schema = ++ Schema.of( ++ Field.of("id", StandardSQLTypeName.INT64), ++ Field.of("amount", StandardSQLTypeName.FLOAT64)); ++ bigQueryResourceManager.createDataset(REGION); ++ bigQueryResourceManager.createTable(TABLE_NAME, schema); ++ ++ // Create Pub/Sub topic and subscription to verify anomaly output. ++ TopicName outputTopic = pubsubResourceManager.createTopic("anomaly-output"); ++ SubscriptionName outputSubscription = ++ pubsubResourceManager.createSubscription(outputTopic, "anomaly-output-sub"); ++ ++ // 1-second fixed windows, MEAN of amount, RobustZScore detector. ++ String metricSpec = ++ "{\"aggregation\":{\"window\":{\"type\":\"fixed\"," ++ + "\"size_seconds\":" ++ + WINDOW_SIZE_SEC ++ + "},\"measures\":[{\"field\":\"amount\"," ++ + "\"agg\":\"MEAN\",\"alias\":\"avg_amount\"}]}}"; ++ String detectorSpec = "{\"type\":\"RobustZScore\"}"; ++ ++ String tableRef = ++ String.format( ++ "%s:%s.%s", ++ bigQueryResourceManager.getProjectId(), ++ bigQueryResourceManager.getDatasetId(), ++ TABLE_NAME); ++ ++ String sinkTableRef = ++ String.format( ++ "%s:%s.%s", ++ bigQueryResourceManager.getProjectId(), ++ bigQueryResourceManager.getDatasetId(), ++ SINK_TABLE_NAME); ++ ++ // --- Act --- ++ ++ // Launch the pipeline first. ++ // start_offset_sec=300 ensures it reads data inserted before and after launch. ++ LaunchConfig.Builder options = ++ LaunchConfig.builder(testName, specPath) ++ .addParameter("table", tableRef) ++ .addParameter("metric_spec", metricSpec) ++ .addParameter("detector_spec", detectorSpec) ++ .addParameter("topic", outputTopic.toString()) ++ .addParameter("poll_interval_sec", "15") ++ .addParameter("start_offset_sec", "300") ++ .addParameter("duration_sec", "600") ++ .addParameter("log_all_results", "true") ++ .addParameter("sink_table", sinkTableRef); ++ ++ LaunchInfo info = launchTemplate(options); ++ assertThatPipeline(info).isRunning(); ++ ++ // Insert baseline data: 360 batches x 100 rows, one batch every second (~6 min). ++ // This gives the pipeline time to start workers and the detector time to warm up ++ // on many 1-second windows of normal data before the anomaly arrives. ++ LOG.info( ++ "Inserting {} batches of {} rows every {}ms (amount={})", ++ BASELINE_BATCHES, ++ ROWS_PER_BATCH, ++ BATCH_INTERVAL_MS, ++ NORMAL_AMOUNT); ++ Random rng = new Random(42); ++ int rowId = 0; ++ for (int batch = 0; batch < BASELINE_BATCHES; batch++) { ++ List rows = new ArrayList<>(); ++ for (int i = 0; i < ROWS_PER_BATCH; i++) { ++ // Tiny variance (stdev ~0.5) so MAD/stdev > 0 for the detector. ++ double amount = NORMAL_AMOUNT + rng.nextGaussian() * 0.5; ++ rows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", amount))); ++ } ++ bigQueryResourceManager.write(TABLE_NAME, rows); ++ if (batch < BASELINE_BATCHES - 1) { ++ TimeUnit.MILLISECONDS.sleep(BATCH_INTERVAL_MS); ++ } ++ if ((batch + 1) % 60 == 0) { ++ LOG.info("Inserted batch {}/{} ({} rows so far)", batch + 1, BASELINE_BATCHES, rowId); ++ } ++ } ++ LOG.info("Inserted {} baseline rows total", rowId); ++ ++ // Pause to ensure the anomaly lands in a clean window, not mixed with baseline. ++ TimeUnit.SECONDS.sleep(2); ++ ++ // Insert anomalous spike: same batch size but 1000x the amount. ++ LOG.info("Inserting anomalous batch ({} rows, amount={})", ROWS_PER_BATCH, ANOMALY_AMOUNT); ++ List anomalyRows = new ArrayList<>(); ++ for (int i = 0; i < ROWS_PER_BATCH; i++) { ++ anomalyRows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", ANOMALY_AMOUNT))); ++ } ++ bigQueryResourceManager.write(TABLE_NAME, anomalyRows); ++ LOG.info("Inserted {} anomalous rows", anomalyRows.size()); ++ ++ // --- Assert --- ++ ++ // Wait for at least 1 anomaly message on the Pub/Sub subscription. ++ PubsubMessagesCheck pubsubCheck = ++ PubsubMessagesCheck.builder(pubsubResourceManager, outputSubscription) ++ .setMinMessages(1) ++ .build(); ++ ++ Result result = pipelineOperator().waitForConditionAndCancel(createConfig(info), pubsubCheck); ++ assertThatResult(result).meetsConditions(); ++ ++ // Verify the anomaly message content. ++ List messages = pubsubCheck.getReceivedMessageList(); ++ assertThat(messages).isNotEmpty(); ++ ++ String messageData = messages.get(0).getMessage().getData().toStringUtf8(); ++ LOG.info("Received anomaly message: {}", messageData); ++ JSONObject payload = new JSONObject(messageData); ++ ++ assertThat(payload.getString("event_description")).contains("Anomaly detected"); ++ assertThat(payload.getString("agent_id")).isEqualTo("RobustZScore"); ++ ++ // --- Verify BQ sink table --- ++ verifySinkTable(SINK_TABLE_NAME, WINDOW_SIZE_SEC, null /* no keys expected */); ++ } ++ ++ /** ++ * Tests a grouped ratio metric (CTR = clicks / impressions) with anomaly detection. ++ * ++ *

Inserts baseline data with a stable click-through rate (~10%) for two campaign types, then ++ * injects an anomalous spike in one campaign's CTR (~90%), and verifies the pipeline detects the ++ * anomaly and publishes it to Pub/Sub with the correct key. ++ */ ++ private void testGroupedRatioMetricImpl() throws IOException, InterruptedException { ++ // --- Arrange --- ++ ++ String tableName = "ctr_test"; ++ ++ Schema schema = ++ Schema.of( ++ Field.of("id", StandardSQLTypeName.INT64), ++ Field.of("campaign_type", StandardSQLTypeName.STRING), ++ Field.of("is_click", StandardSQLTypeName.INT64)); ++ bigQueryResourceManager.createDataset(REGION); ++ bigQueryResourceManager.createTable(tableName, schema); ++ ++ TopicName outputTopic = pubsubResourceManager.createTopic("ctr-anomaly-output"); ++ SubscriptionName outputSubscription = ++ pubsubResourceManager.createSubscription(outputTopic, "ctr-anomaly-output-sub"); ++ ++ String sinkTableName = "ctr_results"; ++ ++ // Grouped ratio: CTR = clicks / impressions per campaign_type. ++ String metricSpec = ++ "{\"aggregation\":{\"window\":{\"type\":\"fixed\"," ++ + "\"size_seconds\":" ++ + WINDOW_SIZE_SEC ++ + "},\"group_by\":[\"campaign_type\"]," ++ + "\"measures\":[{\"field\":\"is_click\",\"agg\":\"SUM\",\"alias\":\"clicks\"}," ++ + "{\"field\":\"is_click\",\"agg\":\"COUNT\",\"alias\":\"impressions\"}]}," ++ + "\"measure_combiner\":{\"expression\":\"clicks / impressions\"}}"; ++ String detectorSpec = "{\"type\":\"RobustZScore\"}"; ++ ++ String tableRef = ++ String.format( ++ "%s:%s.%s", ++ bigQueryResourceManager.getProjectId(), ++ bigQueryResourceManager.getDatasetId(), ++ tableName); ++ ++ String sinkTableRef = ++ String.format( ++ "%s:%s.%s", ++ bigQueryResourceManager.getProjectId(), ++ bigQueryResourceManager.getDatasetId(), ++ sinkTableName); ++ ++ // --- Act --- ++ ++ LaunchConfig.Builder options = ++ LaunchConfig.builder(testName, specPath) ++ .addParameter("table", tableRef) ++ .addParameter("metric_spec", metricSpec) ++ .addParameter("detector_spec", detectorSpec) ++ .addParameter("topic", outputTopic.toString()) ++ .addParameter("poll_interval_sec", "15") ++ .addParameter("start_offset_sec", "300") ++ .addParameter("duration_sec", "600") ++ .addParameter("log_all_results", "true") ++ .addParameter("sink_table", sinkTableRef); ++ ++ LaunchInfo info = launchTemplate(options); ++ assertThatPipeline(info).isRunning(); ++ ++ // Insert baseline data: CTR ~10% for both campaigns. ++ // Campaigns are round-robin (deterministic split), clicks are random ++ // to provide natural per-window variance needed by RobustZScore (MAD > 0). ++ // 500 rows per batch (250 per key) keeps per-window CTR tight (~0.08-0.12). ++ // 360 batches, one batch every second (~6 min). ++ int ctrRowsPerBatch = 500; ++ LOG.info( ++ "Inserting {} batches of {} rows every {}ms (CTR ~10%%)", ++ BASELINE_BATCHES, ctrRowsPerBatch, BATCH_INTERVAL_MS); ++ Random rng = new Random(42); ++ String[] campaigns = {"search", "display"}; ++ int rowId = 0; ++ for (int batch = 0; batch < BASELINE_BATCHES; batch++) { ++ List rows = new ArrayList<>(); ++ for (int i = 0; i < ctrRowsPerBatch; i++) { ++ String campaign = campaigns[i % campaigns.length]; ++ int isClick = rng.nextDouble() < 0.10 ? 1 : 0; ++ rows.add( ++ RowToInsert.of( ++ ImmutableMap.of("id", ++rowId, "campaign_type", campaign, "is_click", isClick))); ++ } ++ bigQueryResourceManager.write(tableName, rows); ++ if (batch < BASELINE_BATCHES - 1) { ++ TimeUnit.MILLISECONDS.sleep(BATCH_INTERVAL_MS); ++ } ++ if ((batch + 1) % 60 == 0) { ++ LOG.info("Inserted batch {}/{} ({} rows so far)", batch + 1, BASELINE_BATCHES, rowId); ++ } ++ } ++ LOG.info("Inserted {} baseline rows total", rowId); ++ ++ TimeUnit.SECONDS.sleep(2); ++ ++ // Inject anomaly: "search" campaign CTR spikes to ~90%. ++ LOG.info("Inserting anomalous batch ({} rows, search CTR ~90%%)", ctrRowsPerBatch); ++ List anomalyRows = new ArrayList<>(); ++ for (int i = 0; i < ctrRowsPerBatch; i++) { ++ int isClick = rng.nextDouble() < 0.90 ? 1 : 0; ++ anomalyRows.add( ++ RowToInsert.of( ++ ImmutableMap.of("id", ++rowId, "campaign_type", "search", "is_click", isClick))); ++ } ++ bigQueryResourceManager.write(tableName, anomalyRows); ++ LOG.info("Inserted {} anomalous rows", anomalyRows.size()); ++ ++ // --- Assert --- ++ ++ PubsubMessagesCheck pubsubCheck = ++ PubsubMessagesCheck.builder(pubsubResourceManager, outputSubscription) ++ .setMinMessages(1) ++ .build(); ++ ++ Result result = pipelineOperator().waitForConditionAndCancel(createConfig(info), pubsubCheck); ++ assertThatResult(result).meetsConditions(); ++ ++ List messages = pubsubCheck.getReceivedMessageList(); ++ assertThat(messages).isNotEmpty(); ++ ++ String messageData = messages.get(0).getMessage().getData().toStringUtf8(); ++ LOG.info("Received CTR anomaly message: {}", messageData); ++ JSONObject payload = new JSONObject(messageData); ++ ++ assertThat(payload.getString("event_description")).contains("Anomaly detected"); ++ assertThat(payload.getString("agent_id")).isEqualTo("RobustZScore"); ++ assertThat(payload.has("key")).isTrue(); ++ ++ // --- Verify BQ sink table --- ++ Set expectedKeys = Set.of("('search',)", "('display',)"); ++ verifySinkTable(sinkTableName, WINDOW_SIZE_SEC, expectedKeys); ++ } ++ ++ /** ++ * Tests the Threshold detector with a fixed expression (value >= 100). ++ * ++ *

Inserts rows with amount=10 (below threshold), then a batch with amount=500 (above ++ * threshold). No warmup period is needed — the threshold fires immediately on any value that ++ * satisfies the expression. ++ */ ++ private void testThresholdDetectorImpl() throws IOException, InterruptedException { ++ // --- Arrange --- ++ ++ String tableName = "threshold_test"; ++ ++ Schema schema = ++ Schema.of( ++ Field.of("id", StandardSQLTypeName.INT64), ++ Field.of("amount", StandardSQLTypeName.FLOAT64)); ++ bigQueryResourceManager.createDataset(REGION); ++ bigQueryResourceManager.createTable(tableName, schema); ++ ++ TopicName outputTopic = pubsubResourceManager.createTopic("threshold-output"); ++ SubscriptionName outputSubscription = ++ pubsubResourceManager.createSubscription(outputTopic, "threshold-output-sub"); ++ ++ String sinkTableName = "threshold_results"; ++ ++ // Simple MEAN metric with Threshold detector. ++ String metricSpec = ++ "{\"aggregation\":{\"window\":{\"type\":\"fixed\"," ++ + "\"size_seconds\":" ++ + WINDOW_SIZE_SEC ++ + "},\"measures\":[{\"field\":\"amount\"," ++ + "\"agg\":\"MEAN\",\"alias\":\"avg_amount\"}]}}"; ++ String detectorSpec = "{\"type\":\"Threshold\",\"expression\":\"value >= 100\"}"; ++ ++ String tableRef = ++ String.format( ++ "%s:%s.%s", ++ bigQueryResourceManager.getProjectId(), ++ bigQueryResourceManager.getDatasetId(), ++ tableName); ++ ++ String sinkTableRef = ++ String.format( ++ "%s:%s.%s", ++ bigQueryResourceManager.getProjectId(), ++ bigQueryResourceManager.getDatasetId(), ++ sinkTableName); ++ ++ // --- Act --- ++ ++ LaunchConfig.Builder options = ++ LaunchConfig.builder(testName, specPath) ++ .addParameter("table", tableRef) ++ .addParameter("metric_spec", metricSpec) ++ .addParameter("detector_spec", detectorSpec) ++ .addParameter("topic", outputTopic.toString()) ++ .addParameter("poll_interval_sec", "15") ++ .addParameter("start_offset_sec", "300") ++ .addParameter("duration_sec", "600") ++ .addParameter("log_all_results", "true") ++ .addParameter("sink_table", sinkTableRef); ++ ++ LaunchInfo info = launchTemplate(options); ++ assertThatPipeline(info).isRunning(); ++ ++ // Insert baseline data: amount=10, well below the threshold of 100. ++ // Fewer batches needed since no warmup — just enough for the pipeline to start. ++ int baselineBatches = 120; ++ LOG.info( ++ "Inserting {} batches of {} rows every {}ms (amount=10, below threshold)", ++ baselineBatches, ++ ROWS_PER_BATCH, ++ BATCH_INTERVAL_MS); ++ int rowId = 0; ++ for (int batch = 0; batch < baselineBatches; batch++) { ++ List rows = new ArrayList<>(); ++ for (int i = 0; i < ROWS_PER_BATCH; i++) { ++ rows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", 10.0))); ++ } ++ bigQueryResourceManager.write(tableName, rows); ++ if (batch < baselineBatches - 1) { ++ TimeUnit.MILLISECONDS.sleep(BATCH_INTERVAL_MS); ++ } ++ if ((batch + 1) % 60 == 0) { ++ LOG.info("Inserted batch {}/{} ({} rows so far)", batch + 1, baselineBatches, rowId); ++ } ++ } ++ LOG.info("Inserted {} baseline rows total", rowId); ++ ++ TimeUnit.SECONDS.sleep(2); ++ ++ // Insert above-threshold batch: amount=500, well above threshold of 100. ++ LOG.info("Inserting above-threshold batch ({} rows, amount=500)", ROWS_PER_BATCH); ++ List alertRows = new ArrayList<>(); ++ for (int i = 0; i < ROWS_PER_BATCH; i++) { ++ alertRows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", 500.0))); ++ } ++ bigQueryResourceManager.write(tableName, alertRows); ++ LOG.info("Inserted {} above-threshold rows", alertRows.size()); ++ ++ // --- Assert --- ++ ++ PubsubMessagesCheck pubsubCheck = ++ PubsubMessagesCheck.builder(pubsubResourceManager, outputSubscription) ++ .setMinMessages(1) ++ .build(); ++ ++ Result result = pipelineOperator().waitForConditionAndCancel(createConfig(info), pubsubCheck); ++ assertThatResult(result).meetsConditions(); ++ ++ List messages = pubsubCheck.getReceivedMessageList(); ++ assertThat(messages).isNotEmpty(); ++ ++ String messageData = messages.get(0).getMessage().getData().toStringUtf8(); ++ LOG.info("Received threshold alert message: {}", messageData); ++ JSONObject payload = new JSONObject(messageData); ++ ++ assertThat(payload.getString("event_description")).contains("Anomaly detected"); ++ assertThat(payload.getString("agent_id")).isEqualTo("Threshold(value >= 100)"); ++ ++ // --- Verify BQ sink table --- ++ verifySinkTable(sinkTableName, WINDOW_SIZE_SEC, null /* no keys expected */); ++ } ++ ++ // ------------------------------------------------------------------------- ++ // Helpers ++ // ------------------------------------------------------------------------- ++ ++ /** ++ * Verifies the BQ sink table written by the pipeline. ++ * ++ * @param tableName sink table name ++ * @param windowSizeSec expected window duration in seconds ++ * @param expectedKeys if non-null, the set of expected key values; if null, key must be absent ++ */ ++ private void verifySinkTable(String tableName, int windowSizeSec, Set expectedKeys) { ++ TableResult tableResult = bigQueryResourceManager.readTable(tableName); ++ List> rows = BigQueryAsserts.tableResultToRecords(tableResult); ++ ++ assertThat(rows).isNotEmpty(); ++ LOG.info("Sink table '{}' has {} rows", tableName, rows.size()); ++ ++ boolean hasOutlier = false; ++ Set observedKeys = new HashSet<>(); ++ Set validLabels = Set.of(-2, 0, 1); ++ ++ for (Map row : rows) { ++ // All expected columns exist. ++ assertThat(row).containsKey("window_start"); ++ assertThat(row).containsKey("window_end"); ++ assertThat(row).containsKey("value"); ++ assertThat(row).containsKey("label"); ++ ++ // Window timestamps parse as valid ISO-8601 UTC. ++ String windowStart = row.get("window_start").toString(); ++ String windowEnd = row.get("window_end").toString(); ++ Instant start = Instant.parse(windowStart); ++ Instant end = Instant.parse(windowEnd); ++ ++ // Window duration matches expected size. ++ long durationSec = end.getEpochSecond() - start.getEpochSecond(); ++ assertThat(durationSec).isEqualTo(windowSizeSec); ++ ++ // Value is a valid number. ++ assertThat(row.get("value")).isNotNull(); ++ double value = ((Number) row.get("value")).doubleValue(); ++ assertThat(Double.isNaN(value)).isFalse(); ++ ++ // Label is valid. ++ int label = ((Number) row.get("label")).intValue(); ++ assertThat(validLabels).contains(label); ++ ++ if (label == 1) { ++ hasOutlier = true; ++ } ++ ++ // Track keys. ++ if (row.containsKey("key") && row.get("key") != null) { ++ observedKeys.add(row.get("key").toString()); ++ } ++ } ++ ++ // At least one outlier was detected. ++ assertThat(hasOutlier).isTrue(); ++ ++ // Verify keys. ++ if (expectedKeys != null) { ++ assertThat(observedKeys).isEqualTo(expectedKeys); ++ } else { ++ assertThat(observedKeys).isEmpty(); ++ } ++ ++ LOG.info( ++ "Sink table '{}' verification passed ({} rows, outlier found)", tableName, rows.size()); ++ } ++} +diff --git a/python/src/test/python/bigquery-anomaly-detection/__init__.py b/python/src/test/python/bigquery-anomaly-detection/__init__.py +new file mode 100644 +index 000000000..e69de29bb +diff --git a/python/src/test/python/bigquery-anomaly-detection/metric_test.py b/python/src/test/python/bigquery-anomaly-detection/metric_test.py +new file mode 100644 +index 000000000..bd6fcd244 +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/metric_test.py +@@ -0,0 +1,243 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Unit tests for bqmonitor.metric.""" ++ ++import json ++import logging ++import unittest ++ ++logging.basicConfig(level=logging.INFO) ++ ++from bqmonitor.metric import AggOp ++from bqmonitor.metric import AggregationSpec ++from bqmonitor.metric import DerivedField ++from bqmonitor.metric import MeasureSpec ++from bqmonitor.metric import MetricSpec ++from bqmonitor.metric import WindowSpec ++from bqmonitor.metric import WindowType ++from bqmonitor.safe_eval import Expr ++ ++ ++class MetricSpecValidationTest(unittest.TestCase): ++ """Tests for MetricSpec validation.""" ++ ++ def _simple_spec(self, **kwargs): ++ defaults = dict( ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=60), ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, ++ alias='total')], ++ )) ++ defaults.update(kwargs) ++ return MetricSpec(**defaults) ++ ++ def test_simple_spec_valid(self): ++ spec = self._simple_spec() ++ self.assertEqual(len(spec.aggregation.measures), 1) ++ ++ def test_no_measures_raises(self): ++ # @specifiable uses lazy init; access an attribute to trigger validation. ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec(aggregation=AggregationSpec(measures=[])) ++ _ = spec.aggregation ++ self.assertIn('at least one measure', str(ctx.exception)) ++ ++ def test_multiple_measures_without_combiner_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='x'), ++ MeasureSpec(field='b', agg=AggOp.COUNT, alias='y'), ++ ])) ++ _ = spec.aggregation ++ self.assertIn('measure_combiner is required', str(ctx.exception)) ++ ++ def test_multiple_measures_with_combiner(self): ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec(field='a', agg=AggOp.COUNT, alias='impressions'), ++ ]), ++ measure_combiner=Expr('clicks / impressions')) ++ self.assertIsNotNone(spec.measure_combiner) ++ ++ def test_combiner_unknown_field_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), ++ ]), ++ measure_combiner=Expr('clicks / impressions')) ++ _ = spec.aggregation ++ self.assertIn('unknown fields', str(ctx.exception)) ++ ++ def test_sliding_without_period_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec(aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.SLIDING, size_seconds=60), ++ measures=[MeasureSpec(field='a', agg=AggOp.SUM, alias='x')])) ++ _ = spec.aggregation ++ self.assertIn('period_seconds', str(ctx.exception)) ++ ++ def test_sliding_with_period(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ window=WindowSpec( ++ type=WindowType.SLIDING, size_seconds=60, period_seconds=10), ++ measures=[MeasureSpec(field='a', agg=AggOp.SUM, alias='x')])) ++ self.assertEqual(spec.aggregation.window.period_seconds, 10) ++ ++ ++class MetricSpecRequiredColumnsTest(unittest.TestCase): ++ """Tests for required_source_columns().""" ++ ++ def test_simple_sum(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) ++ self.assertEqual(spec.required_source_columns(), {'amount'}) ++ ++ def test_count_excludes_field(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[MeasureSpec(field='x', agg=AggOp.COUNT, alias='cnt')])) ++ self.assertEqual(spec.required_source_columns(), set()) ++ ++ def test_group_by_included(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ group_by=['region', 'product'], ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) ++ self.assertEqual( ++ spec.required_source_columns(), {'region', 'product', 'amount'}) ++ ++ def test_derived_field_references(self): ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ measures=[MeasureSpec( ++ field='is_success', agg=AggOp.SUM, alias='successes')]), ++ derived_fields=[ ++ DerivedField( ++ name='is_success', ++ expression=Expr("1 if status == 'ok' else 0")) ++ ]) ++ # 'is_success' is derived, so excluded; 'status' is a source ref. ++ self.assertEqual(spec.required_source_columns(), {'status'}) ++ ++ def test_ctr_metric(self): ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ group_by=['campaign_type'], ++ measures=[ ++ MeasureSpec(field='is_click', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec( ++ field='is_click', agg=AggOp.COUNT, alias='impressions'), ++ ]), ++ measure_combiner=Expr('clicks / impressions')) ++ # is_click (from SUM), campaign_type (from group_by). ++ # COUNT's field is excluded. ++ self.assertEqual( ++ spec.required_source_columns(), {'is_click', 'campaign_type'}) ++ ++ ++class MetricSpecFromDictTest(unittest.TestCase): ++ """Tests for MetricSpec.from_dict() deserialization.""" ++ ++ def test_simple(self): ++ d = { ++ 'aggregation': { ++ 'window': {'type': 'fixed', 'size_seconds': 60}, ++ 'measures': [ ++ {'field': 'amount', 'agg': 'SUM', 'alias': 'total'}], ++ }} ++ spec = MetricSpec.from_dict(d) ++ self.assertEqual(spec.aggregation.window.type, WindowType.FIXED) ++ self.assertEqual(spec.aggregation.window.size_seconds, 60) ++ self.assertEqual(len(spec.aggregation.measures), 1) ++ self.assertEqual(spec.aggregation.measures[0].agg, AggOp.SUM) ++ ++ def test_with_combiner(self): ++ d = { ++ 'aggregation': { ++ 'measures': [ ++ {'field': 'x', 'agg': 'SUM', 'alias': 'clicks'}, ++ {'field': 'x', 'agg': 'COUNT', 'alias': 'impressions'}], ++ }, ++ 'measure_combiner': {'expression': 'clicks / impressions'}, ++ } ++ spec = MetricSpec.from_dict(d) ++ self.assertIsNotNone(spec.measure_combiner) ++ self.assertEqual(spec.measure_combiner.field_refs(), {'clicks', 'impressions'}) ++ ++ def test_with_derived_fields(self): ++ d = { ++ 'aggregation': { ++ 'measures': [ ++ {'field': 'is_ok', 'agg': 'SUM', 'alias': 'ok_count'}], ++ }, ++ 'derived_fields': [ ++ {'name': 'is_ok', 'expression': "1 if status == 'ok' else 0"}], ++ } ++ spec = MetricSpec.from_dict(d) ++ self.assertEqual(len(spec.derived_fields), 1) ++ self.assertEqual(spec.derived_fields[0].name, 'is_ok') ++ ++ def test_roundtrip_json(self): ++ """from_dict(to_dict(spec)) should produce an equivalent spec.""" ++ original = MetricSpec( ++ aggregation=AggregationSpec( ++ window=WindowSpec( ++ type=WindowType.SLIDING, size_seconds=300, period_seconds=60), ++ group_by=['region'], ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec(field='a', agg=AggOp.COUNT, alias='impressions'), ++ ]), ++ measure_combiner=Expr('clicks / impressions'), ++ name='ctr') ++ d = original.to_dict() ++ # Verify JSON-serializable. ++ json_str = json.dumps(d) ++ restored = MetricSpec.from_dict(json.loads(json_str)) ++ self.assertEqual(restored.name, 'ctr') ++ self.assertEqual(restored.aggregation.window.type, WindowType.SLIDING) ++ self.assertEqual(restored.aggregation.window.period_seconds, 60) ++ self.assertEqual(len(restored.aggregation.measures), 2) ++ self.assertEqual( ++ restored.required_source_columns(), {'a', 'region'}) ++ ++ ++class MetricSpecToDictTest(unittest.TestCase): ++ """Tests for MetricSpec.to_dict() serialization.""" ++ ++ def test_simple(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=60), ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) ++ d = spec.to_dict() ++ self.assertEqual(d['aggregation']['window']['type'], 'fixed') ++ self.assertEqual(d['aggregation']['measures'][0]['agg'], 'SUM') ++ ++ def test_excludes_optional_none(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[MeasureSpec(field='x', agg=AggOp.SUM, alias='y')])) ++ d = spec.to_dict() ++ self.assertNotIn('derived_fields', d) ++ self.assertNotIn('measure_combiner', d) ++ self.assertNotIn('name', d) ++ ++ ++if __name__ == '__main__': ++ unittest.main() +diff --git a/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py b/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py +new file mode 100644 +index 000000000..3f654893d +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py +@@ -0,0 +1,287 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Unit tests for bqmonitor.pipeline helpers.""" ++ ++import json ++import logging ++import unittest ++ ++logging.basicConfig(level=logging.INFO) ++ ++import apache_beam as beam ++from apache_beam.ml.anomaly.base import AnomalyPrediction ++from apache_beam.ml.anomaly.base import AnomalyResult ++ ++from bqmonitor.pipeline import _FormatAnomalyAsJson ++from bqmonitor.pipeline import _FormatResultForBQ ++from bqmonitor.pipeline import _parse_detector_spec ++from bqmonitor.pipeline import _parse_table_ref ++from bqmonitor.pipeline import _ThresholdAlert ++from bqmonitor.pipeline import _unpack_result ++ ++ ++class ParseTableRefTest(unittest.TestCase): ++ """Tests for _parse_table_ref().""" ++ ++ def test_colon_format(self): ++ p, d, t = _parse_table_ref('my-project:my_dataset.my_table') ++ self.assertEqual(p, 'my-project') ++ self.assertEqual(d, 'my_dataset') ++ self.assertEqual(t, 'my_table') ++ ++ def test_dot_format(self): ++ p, d, t = _parse_table_ref('my-project.my_dataset.my_table') ++ self.assertEqual(p, 'my-project') ++ self.assertEqual(d, 'my_dataset') ++ self.assertEqual(t, 'my_table') ++ ++ def test_invalid_format_raises(self): ++ with self.assertRaises(ValueError): ++ _parse_table_ref('not_valid') ++ ++ def test_empty_raises(self): ++ with self.assertRaises(ValueError): ++ _parse_table_ref('') ++ ++ ++class UnpackResultTest(unittest.TestCase): ++ """Tests for _unpack_result().""" ++ ++ def test_keyed(self): ++ result = object() ++ key, r = _unpack_result(('mykey', result)) ++ self.assertEqual(key, 'mykey') ++ self.assertIs(r, result) ++ ++ def test_unkeyed(self): ++ result = object() ++ key, r = _unpack_result(result) ++ self.assertIsNone(key) ++ self.assertIs(r, result) ++ ++ ++class ParseDetectorSpecTest(unittest.TestCase): ++ """Tests for _parse_detector_spec().""" ++ ++ def test_zscore(self): ++ detector = _parse_detector_spec('{"type":"ZScore"}') ++ self.assertEqual(type(detector).__name__, 'ZScore') ++ ++ def test_iqr(self): ++ detector = _parse_detector_spec('{"type":"IQR"}') ++ self.assertEqual(type(detector).__name__, 'IQR') ++ ++ def test_robust_zscore(self): ++ detector = _parse_detector_spec('{"type":"RobustZScore"}') ++ self.assertEqual(type(detector).__name__, 'RobustZScore') ++ ++ def test_threshold(self): ++ detector = _parse_detector_spec( ++ '{"type":"Threshold","expression":"value >= 100"}') ++ self.assertIsInstance(detector, _ThresholdAlert) ++ ++ def test_threshold_missing_expression_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec('{"type":"Threshold"}') ++ self.assertIn('expression', str(ctx.exception)) ++ ++ def test_threshold_invalid_expression_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec( ++ '{"type":"Threshold","expression":"import os"}') ++ self.assertIn('Invalid threshold expression', str(ctx.exception)) ++ ++ def test_unknown_type_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec('{"type":"Unknown"}') ++ self.assertIn('Unknown', str(ctx.exception)) ++ ++ def test_invalid_json_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec('{bad json}') ++ self.assertIn('Invalid JSON', str(ctx.exception)) ++ ++ def test_missing_type_raises(self): ++ with self.assertRaises(ValueError): ++ _parse_detector_spec('{"config":{}}') ++ ++ def test_zscore_with_threshold(self): ++ spec = json.dumps({ ++ 'type': 'ZScore', ++ 'config': { ++ 'threshold_criterion': { ++ 'type': 'FixedThreshold', ++ 'config': {'cutoff': 10}}}}) ++ detector = _parse_detector_spec(spec) ++ self.assertEqual(type(detector).__name__, 'ZScore') ++ ++ ++class ThresholdAlertTest(unittest.TestCase): ++ """Tests for _ThresholdAlert DoFn.""" ++ ++ def _make_row(self, value): ++ return beam.Row(value=value, window_start=0.0, window_end=1.0) ++ ++ def _run_dofn(self, expression, element): ++ dofn = _ThresholdAlert(expression) ++ dofn.setup() ++ return list(dofn.process(element)) ++ ++ def test_above_threshold(self): ++ results = self._run_dofn('value >= 100', self._make_row(500.0)) ++ self.assertEqual(len(results), 1) ++ result = results[0] ++ self.assertIsInstance(result, AnomalyResult) ++ self.assertEqual(result.predictions[0].label, 1) ++ self.assertIsNone(result.predictions[0].score) ++ self.assertEqual( ++ result.predictions[0].model_id, 'Threshold(value >= 100)') ++ ++ def test_below_threshold(self): ++ results = self._run_dofn('value >= 100', self._make_row(50.0)) ++ self.assertEqual(len(results), 1) ++ self.assertEqual(results[0].predictions[0].label, 0) ++ ++ def test_keyed_element(self): ++ row = self._make_row(200.0) ++ results = self._run_dofn('value >= 100', ('mykey', row)) ++ self.assertEqual(len(results), 1) ++ key, result = results[0] ++ self.assertEqual(key, 'mykey') ++ self.assertEqual(result.predictions[0].label, 1) ++ ++ def test_range_expression(self): ++ dofn = _ThresholdAlert('value > 100 or value < -100') ++ dofn.setup() ++ ++ # Above range. ++ results = list(dofn.process(self._make_row(200.0))) ++ self.assertEqual(results[0].predictions[0].label, 1) ++ ++ # Below range. ++ results = list(dofn.process(self._make_row(-200.0))) ++ self.assertEqual(results[0].predictions[0].label, 1) ++ ++ # Within range. ++ results = list(dofn.process(self._make_row(50.0))) ++ self.assertEqual(results[0].predictions[0].label, 0) ++ ++ def test_less_than_threshold(self): ++ results = self._run_dofn('value <= 0.01', self._make_row(0.005)) ++ self.assertEqual(results[0].predictions[0].label, 1) ++ ++ results = self._run_dofn('value <= 0.01', self._make_row(0.5)) ++ self.assertEqual(results[0].predictions[0].label, 0) ++ ++ ++class FormatAnomalyAsJsonTest(unittest.TestCase): ++ """Tests for _FormatAnomalyAsJson DoFn.""" ++ ++ def _make_result(self, label, value=42.0, score=5.0, model_id='TestModel'): ++ row = beam.Row(value=value, window_start=1000.0, window_end=1001.0) ++ prediction = AnomalyPrediction( ++ model_id=model_id, score=score, label=label) ++ return AnomalyResult(example=row, predictions=[prediction]) ++ ++ def test_outlier_emits_json(self): ++ dofn = _FormatAnomalyAsJson() ++ results = list(dofn.process(self._make_result(label=1))) ++ self.assertEqual(len(results), 1) ++ payload = json.loads(results[0]) ++ self.assertIn('Anomaly detected', payload['event_description']) ++ self.assertEqual(payload['agent_id'], 'TestModel') ++ ++ def test_normal_emits_nothing(self): ++ dofn = _FormatAnomalyAsJson() ++ results = list(dofn.process(self._make_result(label=0))) ++ self.assertEqual(len(results), 0) ++ ++ def test_warmup_emits_nothing(self): ++ dofn = _FormatAnomalyAsJson() ++ results = list(dofn.process(self._make_result(label=-2))) ++ self.assertEqual(len(results), 0) ++ ++ def test_keyed_outlier_includes_key(self): ++ dofn = _FormatAnomalyAsJson() ++ result = self._make_result(label=1) ++ outputs = list(dofn.process(('campaign_search', result))) ++ self.assertEqual(len(outputs), 1) ++ payload = json.loads(outputs[0]) ++ self.assertEqual(payload['key'], 'campaign_search') ++ ++ def test_threshold_model_id(self): ++ dofn = _FormatAnomalyAsJson() ++ result = self._make_result( ++ label=1, model_id='Threshold(value >= 100)') ++ outputs = list(dofn.process(result)) ++ payload = json.loads(outputs[0]) ++ self.assertEqual(payload['agent_id'], 'Threshold(value >= 100)') ++ ++ ++class FormatResultForBQTest(unittest.TestCase): ++ """Tests for _FormatResultForBQ DoFn.""" ++ ++ def _make_result(self, label, value=42.0, score=5.0): ++ row = beam.Row(value=value, window_start=1000.0, window_end=1001.0) ++ prediction = AnomalyPrediction( ++ model_id='TestModel', score=score, label=label) ++ return AnomalyResult(example=row, predictions=[prediction]) ++ ++ def test_outlier_row(self): ++ dofn = _FormatResultForBQ() ++ results = list(dofn.process(self._make_result(label=1, value=99.0, ++ score=4.5))) ++ self.assertEqual(len(results), 1) ++ row = results[0] ++ self.assertAlmostEqual(row['value'], 99.0) ++ self.assertAlmostEqual(row['score'], 4.5) ++ self.assertEqual(row['label'], 1) ++ self.assertIn('window_start', row) ++ self.assertIn('window_end', row) ++ self.assertNotIn('key', row) ++ ++ def test_normal_row(self): ++ dofn = _FormatResultForBQ() ++ results = list(dofn.process(self._make_result(label=0))) ++ self.assertEqual(len(results), 1) ++ self.assertEqual(results[0]['label'], 0) ++ ++ def test_warmup_row(self): ++ dofn = _FormatResultForBQ() ++ results = list(dofn.process(self._make_result(label=-2))) ++ self.assertEqual(len(results), 1) ++ self.assertEqual(results[0]['label'], -2) ++ ++ def test_keyed_row_includes_key(self): ++ dofn = _FormatResultForBQ() ++ result = self._make_result(label=1) ++ outputs = list(dofn.process(('campaign_search', result))) ++ self.assertEqual(len(outputs), 1) ++ self.assertEqual(outputs[0]['key'], 'campaign_search') ++ ++ def test_none_score(self): ++ row = beam.Row(value=10.0, window_start=0.0, window_end=1.0) ++ prediction = AnomalyPrediction( ++ model_id='Test', score=None, label=0) ++ result = AnomalyResult(example=row, predictions=[prediction]) ++ dofn = _FormatResultForBQ() ++ outputs = list(dofn.process(result)) ++ self.assertIsNone(outputs[0]['score']) ++ ++ ++if __name__ == '__main__': ++ unittest.main() +diff --git a/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt b/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt +new file mode 100644 +index 000000000..57726014e +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt +@@ -0,0 +1 @@ ++apache-beam[gcp,test] +diff --git a/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py b/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py +new file mode 100644 +index 000000000..974b10511 +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py +@@ -0,0 +1,244 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Unit tests for bqmonitor.safe_eval.""" ++ ++import logging ++import unittest ++ ++logging.basicConfig(level=logging.INFO) ++ ++from bqmonitor.safe_eval import Expr ++ ++ ++class ExprArithmeticTest(unittest.TestCase): ++ """Tests for arithmetic operations.""" ++ ++ def test_addition(self): ++ self.assertEqual(Expr('a + b')({'a': 3, 'b': 4}), 7) ++ ++ def test_subtraction(self): ++ self.assertEqual(Expr('a - b')({'a': 10, 'b': 3}), 7) ++ ++ def test_multiplication(self): ++ self.assertEqual(Expr('a * b')({'a': 5, 'b': 6}), 30) ++ ++ def test_division(self): ++ self.assertAlmostEqual(Expr('a / b')({'a': 10, 'b': 3}), 10 / 3) ++ ++ def test_floor_division(self): ++ self.assertEqual(Expr('a // b')({'a': 10, 'b': 3}), 3) ++ ++ def test_modulo(self): ++ self.assertEqual(Expr('a % b')({'a': 10, 'b': 3}), 1) ++ ++ def test_power(self): ++ self.assertEqual(Expr('x ** 2')({'x': 5}), 25) ++ ++ def test_negation(self): ++ self.assertEqual(Expr('-x')({'x': 7}), -7) ++ ++ def test_parentheses(self): ++ self.assertEqual(Expr('(a + b) * c')({'a': 2, 'b': 3, 'c': 4}), 20) ++ ++ ++class ExprComparisonTest(unittest.TestCase): ++ """Tests for comparison operations.""" ++ ++ def test_eq(self): ++ self.assertTrue(Expr('x == 1')({'x': 1})) ++ self.assertFalse(Expr('x == 1')({'x': 2})) ++ ++ def test_neq(self): ++ self.assertTrue(Expr('x != 1')({'x': 2})) ++ ++ def test_lt(self): ++ self.assertTrue(Expr('x < 5')({'x': 3})) ++ self.assertFalse(Expr('x < 5')({'x': 5})) ++ ++ def test_lte(self): ++ self.assertTrue(Expr('x <= 5')({'x': 5})) ++ ++ def test_gt(self): ++ self.assertTrue(Expr('x > 5')({'x': 6})) ++ ++ def test_gte(self): ++ self.assertTrue(Expr('x >= 5')({'x': 5})) ++ ++ def test_string_comparison(self): ++ self.assertTrue(Expr("s == 'ok'")({'s': 'ok'})) ++ self.assertFalse(Expr("s == 'ok'")({'s': 'fail'})) ++ ++ ++class ExprBooleanTest(unittest.TestCase): ++ """Tests for boolean logic.""" ++ ++ def test_and(self): ++ self.assertTrue(Expr('a > 0 and b > 0')({'a': 1, 'b': 1})) ++ self.assertFalse(Expr('a > 0 and b > 0')({'a': 1, 'b': -1})) ++ ++ def test_or(self): ++ self.assertTrue(Expr('a > 0 or b > 0')({'a': -1, 'b': 1})) ++ self.assertFalse(Expr('a > 0 or b > 0')({'a': -1, 'b': -1})) ++ ++ def test_not(self): ++ self.assertTrue(Expr('not failed')({'failed': False})) ++ self.assertFalse(Expr('not failed')({'failed': True})) ++ ++ def test_compound(self): ++ e = Expr("x > 0 and not disabled or override == 'yes'") ++ self.assertTrue(e({'x': 1, 'disabled': False, 'override': 'no'})) ++ self.assertTrue(e({'x': -1, 'disabled': True, 'override': 'yes'})) ++ self.assertFalse(e({'x': -1, 'disabled': True, 'override': 'no'})) ++ ++ ++class ExprIfElseTest(unittest.TestCase): ++ """Tests for conditional expressions.""" ++ ++ def test_if_else(self): ++ e = Expr("1 if status == 'ok' else 0") ++ self.assertEqual(e({'status': 'ok'}), 1) ++ self.assertEqual(e({'status': 'fail'}), 0) ++ ++ def test_if_else_with_bool(self): ++ e = Expr("1 if a > 0 and b > 0 else 0") ++ self.assertEqual(e({'a': 1, 'b': 1}), 1) ++ self.assertEqual(e({'a': 1, 'b': -1}), 0) ++ ++ ++class ExprBuiltinsTest(unittest.TestCase): ++ """Tests for safe builtin functions.""" ++ ++ def test_abs(self): ++ self.assertEqual(Expr('abs(x)')({'x': -7}), 7) ++ self.assertEqual(Expr('abs(x)')({'x': 7}), 7) ++ ++ def test_min(self): ++ self.assertEqual(Expr('min(a, b)')({'a': 3, 'b': 5}), 3) ++ ++ def test_max(self): ++ self.assertEqual(Expr('max(a, b)')({'a': 3, 'b': 5}), 5) ++ ++ def test_max_with_literal(self): ++ self.assertEqual(Expr('max(x, 1)')({'x': 0}), 1) ++ ++ def test_round(self): ++ self.assertAlmostEqual(Expr('round(x, 2)')({'x': 3.14159}), 3.14) ++ ++ def test_nested_builtins(self): ++ self.assertEqual(Expr('max(abs(a), abs(b))')({'a': -5, 'b': 3}), 5) ++ ++ ++class ExprFieldRefsTest(unittest.TestCase): ++ """Tests for field_refs() extraction.""" ++ ++ def test_simple(self): ++ self.assertEqual(Expr('a + b').field_refs(), {'a', 'b'}) ++ ++ def test_excludes_builtins(self): ++ self.assertEqual( ++ Expr('max(clicks, 1) / abs(total)').field_refs(), ++ {'clicks', 'total'}) ++ ++ def test_if_else_refs(self): ++ self.assertEqual( ++ Expr("1 if status == 'ok' else 0").field_refs(), {'status'}) ++ ++ def test_no_refs(self): ++ self.assertEqual(Expr('1 + 2').field_refs(), set()) ++ ++ ++class ExprValidationTest(unittest.TestCase): ++ """Tests for expression validation / rejection.""" ++ ++ def test_rejects_import(self): ++ with self.assertRaises((ValueError, SyntaxError)): ++ Expr('__import__("os")') ++ ++ def test_rejects_unknown_function(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('eval("bad")') ++ self.assertIn('eval', str(ctx.exception)) ++ ++ def test_rejects_attribute_access(self): ++ with self.assertRaises(ValueError): ++ Expr('x.__class__') ++ ++ def test_rejects_subscript(self): ++ with self.assertRaises(ValueError): ++ Expr('x[0]') ++ ++ def test_rejects_lambda(self): ++ with self.assertRaises((ValueError, SyntaxError)): ++ Expr('lambda x: x') ++ ++ def test_rejects_chained_comparison(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('a < b < c') ++ self.assertIn('Chained', str(ctx.exception)) ++ ++ def test_rejects_keyword_args(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('round(x, ndigits=2)') ++ self.assertIn('Keyword', str(ctx.exception)) ++ ++ def test_rejects_unsupported_literal(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('b"bytes"') ++ self.assertIn('bytes', str(ctx.exception)) ++ ++ ++class ExprPickleTest(unittest.TestCase): ++ """Tests for pickle/unpickle support.""" ++ ++ def test_roundtrip(self): ++ import pickle ++ original = Expr('a + b') ++ restored = pickle.loads(pickle.dumps(original)) ++ self.assertEqual(original, restored) ++ self.assertEqual(restored({'a': 1, 'b': 2}), 3) ++ ++ ++class ExprRealWorldTest(unittest.TestCase): ++ """Tests using realistic metric expressions.""" ++ ++ def test_ctr(self): ++ e = Expr('clicks / impressions') ++ self.assertAlmostEqual(e({'clicks': 50, 'impressions': 1000}), 0.05) ++ ++ def test_safe_ctr(self): ++ e = Expr('clicks / max(impressions, 1)') ++ self.assertEqual(e({'clicks': 0, 'impressions': 0}), 0.0) ++ ++ def test_derived_field(self): ++ e = Expr("1 if status == 'success' else 0") ++ self.assertEqual(e({'status': 'success'}), 1) ++ self.assertEqual(e({'status': 'error'}), 0) ++ ++ def test_threshold_expression(self): ++ e = Expr('value >= 100') ++ self.assertTrue(e({'value': 500})) ++ self.assertFalse(e({'value': 50})) ++ ++ def test_range_threshold(self): ++ e = Expr('value > 100 or value < -100') ++ self.assertTrue(e({'value': 200})) ++ self.assertTrue(e({'value': -200})) ++ self.assertFalse(e({'value': 50})) ++ ++ ++if __name__ == '__main__': ++ unittest.main() diff --git a/launcher_logs.txt b/launcher_logs.txt new file mode 100644 index 0000000000..b1cbdb3030 --- /dev/null +++ b/launcher_logs.txt @@ -0,0 +1,15 @@ +ps aux | grep python +root 901 0.1 1.0 42888 37776 ? Ss 02:50 0:00 /usr/bin/python3.11 /usr/lib/python-exec/python3.11/cloud-init modules --mode=final +root 1086 0.0 0.6 1623628 25588 ? Ssl 02:50 0:00 /usr/bin/docker run --entrypoint /opt/google/dataflow/python_template_launcher --restart on-failure:3 --net=host -v /var/log/dataflow/template_launcher:/var/log/dataflow/template_launcher gcr.io/dataflow-twest/bigquery-anomaly-detection:templates +root 1193 0.0 1.4 1821168 53360 ? Ssl 02:52 0:00 /opt/google/dataflow/python_template_launcher +root 1212 1.5 5.6 480000 211060 ? Sl 02:52 0:03 python main.py --requirements_file=requirements.txt --setup_file=setup.py --detector_spec={"type":"ZScore"} --runner=DataflowRunner --job_name=bq-anomaly-detection-20260308-224948 --metric_spec={"aggregation":{"window":{"type":"fixed","size_seconds":60},"measures":[{"field":"amount","agg":"SUM","alias":"revenue"}]}} --table=dataflow-twest:cdc.cuj1_revenue --duration_sec=60000 --project=dataflow-twest --template_location=gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_49_49-9917035267881694824/job_object --region=us-central1 --service_account_email=672035346122-compute@developer.gserviceaccount.com --temp_location=gs://dataflow-staging-us-central1-672035346122/tmp --staging_location=gs://dataflow-staging-us-central1-672035346122/staging --labels=goog-dataflow-provided-template-name=bigquery_anomaly_detection,goog-dataflow-provided-template-type=flex,goog-dataflow-provided-template-version=templates --labels=goog-dataflow-provided-template-version=templates --labels=goog-dataflow-provided-template-name=bigquery_anomaly_detection --labels=goog-dataflow-provided-template-type=flex +root 1214 3.7 8.2 389460 309348 ? S 02:52 0:09 /usr/local/bin/python -m pip download --dest /tmp/dataflow-requirements-cache -r /tmp/tmp2s94ukxm/tmp_requirements.txt --exists-action i --no-deps --implementation cp --abi cp311 --platform manylinux2014_x86_64 +root 1235 0.0 0.6 29432 24540 ? S 02:52 0:00 /usr/local/bin/python /usr/local/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py prepare_metadata_for_build_wheel /tmp/tmp9yjgqxts +root 2696 0.0 0.0 2584 1632 ? S 02:56 0:00 /bin/sh -c ccache cc -Inumpy/_core/libloops_comparison.dispatch.h_baseline.a.p -Inumpy/_core -I../numpy/_core -Inumpy/_core/include -I../numpy/_core/include -I../numpy/_core/src/common -I../numpy/_core/src/multiarray -I../numpy/_core/src/npymath -I../numpy/_core/src/umath -I../numpy/_core/src/highway -I/usr/local/include/python3.11 -I/tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -fdiagnostics-color=always -DNDEBUG -Wall -Winvalid-pch -std=c11 -O3 -fno-strict-aliasing -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -fPIC -DNPY_INTERNAL_BUILD -DHAVE_NPY_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 -O3 -DNPY_HAVE_X86_V2 -DNPY_HAVE_SSE -DNPY_HAVE_SSE2 -DNPY_HAVE_SSE3 -DNPY_HAVE_SSSE3 -DNPY_HAVE_SSE41 -DNPY_HAVE_SSE42 -DNPY_HAVE_POPCNT -DNPY_HAVE_LAHF -DNPY_HAVE_CX16 -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -DHWY_WANT_SSE4 -DHWY_DISABLE_PCLMUL_AES -DNPY_MTARGETS_BASELINE -MD -MQ numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o -MF numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o.d -o numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o -c numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/loops_comparison.dispatch.c +root 2697 0.6 0.1 8188 4908 ? S 02:56 0:00 ccache cc -Inumpy/_core/libloops_comparison.dispatch.h_baseline.a.p -Inumpy/_core -I../numpy/_core -Inumpy/_core/include -I../numpy/_core/include -I../numpy/_core/src/common -I../numpy/_core/src/multiarray -I../numpy/_core/src/npymath -I../numpy/_core/src/umath -I../numpy/_core/src/highway -I/usr/local/include/python3.11 -I/tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -fdiagnostics-color=always -DNDEBUG -Wall -Winvalid-pch -std=c11 -O3 -fno-strict-aliasing -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -fPIC -DNPY_INTERNAL_BUILD -DHAVE_NPY_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 -O3 -DNPY_HAVE_X86_V2 -DNPY_HAVE_SSE -DNPY_HAVE_SSE2 -DNPY_HAVE_SSE3 -DNPY_HAVE_SSSE3 -DNPY_HAVE_SSE41 -DNPY_HAVE_SSE42 -DNPY_HAVE_POPCNT -DNPY_HAVE_LAHF -DNPY_HAVE_CX16 -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -DHWY_WANT_SSE4 -DHWY_DISABLE_PCLMUL_AES -DNPY_MTARGETS_BASELINE -MD -MQ numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o -MF numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o.d -o numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o -c numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/loops_comparison.dispatch.c +root 2700 0.0 0.0 4248 2596 ? S 02:56 0:00 /usr/bin/cc -Wall -Winvalid-pch -std=c11 -O3 -fno-strict-aliasing -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -fPIC -O3 -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -fdiagnostics-color=always -Inumpy/_core/libloops_comparison.dispatch.h_baseline.a.p -Inumpy/_core -I../numpy/_core -Inumpy/_core/include -I../numpy/_core/include -I../numpy/_core/src/common -I../numpy/_core/src/multiarray -I../numpy/_core/src/npymath -I../numpy/_core/src/umath -I../numpy/_core/src/highway -I/usr/local/include/python3.11 -I/tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -DNDEBUG -DNPY_INTERNAL_BUILD -DHAVE_NPY_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 -DNPY_HAVE_X86_V2 -DNPY_HAVE_SSE -DNPY_HAVE_SSE2 -DNPY_HAVE_SSE3 -DNPY_HAVE_SSSE3 -DNPY_HAVE_SSE41 -DNPY_HAVE_SSE42 -DNPY_HAVE_POPCNT -DNPY_HAVE_LAHF -DNPY_HAVE_CX16 -DHWY_WANT_SSE4 -DHWY_DISABLE_PCLMUL_AES -DNPY_MTARGETS_BASELINE -c -MD -MQ numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o -MF numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o.d -fdiagnostics-color -o numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/loops_comparison.dispatch.c +root 2701 49.8 3.3 141016 124340 ? R 02:56 0:03 /usr/lib/gcc/x86_64-linux-gnu/12/cc1 -quiet -I numpy/_core/libloops_comparison.dispatch.h_baseline.a.p -I numpy/_core -I ../numpy/_core -I numpy/_core/include -I ../numpy/_core/include -I ../numpy/_core/src/common -I ../numpy/_core/src/multiarray -I ../numpy/_core/src/npymath -I ../numpy/_core/src/umath -I ../numpy/_core/src/highway -I /usr/local/include/python3.11 -I /tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -imultiarch x86_64-linux-gnu -MD numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.d -MF numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o.d -MQ numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/meson-generated_loops_comparison.dispatch.c.o -D NDEBUG -D NPY_INTERNAL_BUILD -D HAVE_NPY_CONFIG_H -D _FILE_OFFSET_BITS=64 -D _LARGEFILE_SOURCE=1 -D _LARGEFILE64_SOURCE=1 -D NPY_HAVE_X86_V2 -D NPY_HAVE_SSE -D NPY_HAVE_SSE2 -D NPY_HAVE_SSE3 -D NPY_HAVE_SSSE3 -D NPY_HAVE_SSE41 -D NPY_HAVE_SSE42 -D NPY_HAVE_POPCNT -D NPY_HAVE_LAHF -D NPY_HAVE_CX16 -D HWY_WANT_SSE4 -D HWY_DISABLE_PCLMUL_AES -D NPY_MTARGETS_BASELINE numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/loops_comparison.dispatch.c -quiet -dumpdir numpy/_core/libloops_comparison.dispatch.h_baseline.a.p/ -dumpbase meson-generated_loops_comparison.dispatch.c.c -dumpbase-ext .c -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -mtune=generic -march=x86-64 -O3 -O3 -Wall -Winvalid-pch -std=c11 -fdiagnostics-color=always -fno-strict-aliasing -fPIC -fasynchronous-unwind-tables -o /tmp/cctOHmFb.s +root 2703 0.0 0.0 2584 1680 ? S 02:56 0:00 /bin/sh -c ccache cc -Inumpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p -Inumpy/_core -I../numpy/_core -Inumpy/_core/include -I../numpy/_core/include -I../numpy/_core/src/common -I../numpy/_core/src/multiarray -I../numpy/_core/src/npymath -I../numpy/_core/src/umath -I../numpy/_core/src/highway -I/usr/local/include/python3.11 -I/tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -fdiagnostics-color=always -DNDEBUG -Wall -Winvalid-pch -std=c11 -O3 -fno-strict-aliasing -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -fPIC -DNPY_INTERNAL_BUILD -DHAVE_NPY_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 -O3 -DNPY_HAVE_X86_V2 -DNPY_HAVE_SSE -DNPY_HAVE_SSE2 -DNPY_HAVE_SSE3 -DNPY_HAVE_SSSE3 -DNPY_HAVE_SSE41 -DNPY_HAVE_SSE42 -DNPY_HAVE_POPCNT -DNPY_HAVE_LAHF -DNPY_HAVE_CX16 -DNPY_HAVE_X86_V3 -DNPY_HAVE_AVX -DNPY_HAVE_AVX2 -DNPY_HAVE_FMA3 -DNPY_HAVE_BMI -DNPY_HAVE_BMI2 -DNPY_HAVE_LZCNT -DNPY_HAVE_F16C -DNPY_HAVE_MOVBE -DNPY_HAVE_X86_V4 -DNPY_HAVE_AVX512F -DNPY_HAVE_AVX512CD -DNPY_HAVE_AVX512VL -DNPY_HAVE_AVX512BW -DNPY_HAVE_AVX512DQ -DNPY_HAVE_AVX512_SKX -DNPY_HAVE_AVX512F_REDUCE -DNPY_HAVE_AVX512BW_MASK -DNPY_HAVE_AVX512DQ_MASK -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -DHWY_WANT_SSE4 -DHWY_DISABLE_PCLMUL_AES -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mf16c -mmovbe -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -DNPY_MTARGETS_CURRENT=X86_V4 -MD -MQ numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o -MF numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o.d -o numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o -c numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/loops_comparison.dispatch.c +root 2704 1.0 0.1 8440 5052 ? S 02:56 0:00 ccache cc -Inumpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p -Inumpy/_core -I../numpy/_core -Inumpy/_core/include -I../numpy/_core/include -I../numpy/_core/src/common -I../numpy/_core/src/multiarray -I../numpy/_core/src/npymath -I../numpy/_core/src/umath -I../numpy/_core/src/highway -I/usr/local/include/python3.11 -I/tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -fdiagnostics-color=always -DNDEBUG -Wall -Winvalid-pch -std=c11 -O3 -fno-strict-aliasing -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -fPIC -DNPY_INTERNAL_BUILD -DHAVE_NPY_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 -O3 -DNPY_HAVE_X86_V2 -DNPY_HAVE_SSE -DNPY_HAVE_SSE2 -DNPY_HAVE_SSE3 -DNPY_HAVE_SSSE3 -DNPY_HAVE_SSE41 -DNPY_HAVE_SSE42 -DNPY_HAVE_POPCNT -DNPY_HAVE_LAHF -DNPY_HAVE_CX16 -DNPY_HAVE_X86_V3 -DNPY_HAVE_AVX -DNPY_HAVE_AVX2 -DNPY_HAVE_FMA3 -DNPY_HAVE_BMI -DNPY_HAVE_BMI2 -DNPY_HAVE_LZCNT -DNPY_HAVE_F16C -DNPY_HAVE_MOVBE -DNPY_HAVE_X86_V4 -DNPY_HAVE_AVX512F -DNPY_HAVE_AVX512CD -DNPY_HAVE_AVX512VL -DNPY_HAVE_AVX512BW -DNPY_HAVE_AVX512DQ -DNPY_HAVE_AVX512_SKX -DNPY_HAVE_AVX512F_REDUCE -DNPY_HAVE_AVX512BW_MASK -DNPY_HAVE_AVX512DQ_MASK -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -DHWY_WANT_SSE4 -DHWY_DISABLE_PCLMUL_AES -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mf16c -mmovbe -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -DNPY_MTARGETS_CURRENT=X86_V4 -MD -MQ numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o -MF numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o.d -o numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o -c numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/loops_comparison.dispatch.c +root 2707 0.0 0.0 4248 2660 ? S 02:56 0:00 /usr/bin/cc -Wall -Winvalid-pch -std=c11 -O3 -fno-strict-aliasing -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -fPIC -O3 -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mf16c -mmovbe -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -fdiagnostics-color=always -Inumpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p -Inumpy/_core -I../numpy/_core -Inumpy/_core/include -I../numpy/_core/include -I../numpy/_core/src/common -I../numpy/_core/src/multiarray -I../numpy/_core/src/npymath -I../numpy/_core/src/umath -I../numpy/_core/src/highway -I/usr/local/include/python3.11 -I/tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -DNDEBUG -DNPY_INTERNAL_BUILD -DHAVE_NPY_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 -DNPY_HAVE_X86_V2 -DNPY_HAVE_SSE -DNPY_HAVE_SSE2 -DNPY_HAVE_SSE3 -DNPY_HAVE_SSSE3 -DNPY_HAVE_SSE41 -DNPY_HAVE_SSE42 -DNPY_HAVE_POPCNT -DNPY_HAVE_LAHF -DNPY_HAVE_CX16 -DNPY_HAVE_X86_V3 -DNPY_HAVE_AVX -DNPY_HAVE_AVX2 -DNPY_HAVE_FMA3 -DNPY_HAVE_BMI -DNPY_HAVE_BMI2 -DNPY_HAVE_LZCNT -DNPY_HAVE_F16C -DNPY_HAVE_MOVBE -DNPY_HAVE_X86_V4 -DNPY_HAVE_AVX512F -DNPY_HAVE_AVX512CD -DNPY_HAVE_AVX512VL -DNPY_HAVE_AVX512BW -DNPY_HAVE_AVX512DQ -DNPY_HAVE_AVX512_SKX -DNPY_HAVE_AVX512F_REDUCE -DNPY_HAVE_AVX512BW_MASK -DNPY_HAVE_AVX512DQ_MASK -DHWY_WANT_SSE4 -DHWY_DISABLE_PCLMUL_AES -DNPY_MTARGETS_CURRENT=X86_V4 -c -MD -MQ numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o -MF numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o.d -fdiagnostics-color -o numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/loops_comparison.dispatch.c +root 2708 49.4 3.9 207320 149692 ? R 02:56 0:02 /usr/lib/gcc/x86_64-linux-gnu/12/cc1 -quiet -I numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p -I numpy/_core -I ../numpy/_core -I numpy/_core/include -I ../numpy/_core/include -I ../numpy/_core/src/common -I ../numpy/_core/src/multiarray -I ../numpy/_core/src/npymath -I ../numpy/_core/src/umath -I ../numpy/_core/src/highway -I /usr/local/include/python3.11 -I /tmp/pip-download-pf2q5zl5/numpy_f89d373ccf9946938212a2e96e82396b/.mesonpy-vx2nouch/meson_cpu -imultiarch x86_64-linux-gnu -MD numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.d -MF numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o.d -MQ numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/meson-generated_loops_comparison.dispatch.c.o -D NDEBUG -D NPY_INTERNAL_BUILD -D HAVE_NPY_CONFIG_H -D _FILE_OFFSET_BITS=64 -D _LARGEFILE_SOURCE=1 -D _LARGEFILE64_SOURCE=1 -D NPY_HAVE_X86_V2 -D NPY_HAVE_SSE -D NPY_HAVE_SSE2 -D NPY_HAVE_SSE3 -D NPY_HAVE_SSSE3 -D NPY_HAVE_SSE41 -D NPY_HAVE_SSE42 -D NPY_HAVE_POPCNT -D NPY_HAVE_LAHF -D NPY_HAVE_CX16 -D NPY_HAVE_X86_V3 -D NPY_HAVE_AVX -D NPY_HAVE_AVX2 -D NPY_HAVE_FMA3 -D NPY_HAVE_BMI -D NPY_HAVE_BMI2 -D NPY_HAVE_LZCNT -D NPY_HAVE_F16C -D NPY_HAVE_MOVBE -D NPY_HAVE_X86_V4 -D NPY_HAVE_AVX512F -D NPY_HAVE_AVX512CD -D NPY_HAVE_AVX512VL -D NPY_HAVE_AVX512BW -D NPY_HAVE_AVX512DQ -D NPY_HAVE_AVX512_SKX -D NPY_HAVE_AVX512F_REDUCE -D NPY_HAVE_AVX512BW_MASK -D NPY_HAVE_AVX512DQ_MASK -D HWY_WANT_SSE4 -D HWY_DISABLE_PCLMUL_AES -D NPY_MTARGETS_CURRENT=X86_V4 numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/loops_comparison.dispatch.c -quiet -dumpdir numpy/_core/libloops_comparison.dispatch.h_X86_V4.a.p/ -dumpbase meson-generated_loops_comparison.dispatch.c.c -dumpbase-ext .c -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -msahf -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mf16c -mmovbe -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mtune=generic -march=x86-64 -O3 -O3 -Wall -Winvalid-pch -std=c11 -fdiagnostics-color=always -fno-strict-aliasing -fPIC -fasynchronous-unwind-tables -o /tmp/ccunEFXA.s \ No newline at end of file diff --git a/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java b/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java index 9d96e4ecff..f4e68bd239 100644 --- a/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java +++ b/plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java @@ -210,6 +210,9 @@ public Builder( this.parameters.put("filesToCopy", ""); this.parameters.put("directoriesToCopy", ""); + this.parameters.put("requirementsFile", "requirements.txt"); + this.parameters.put("setupFileEnv", ""); + this.parameters.put("setupInstall", ""); this.parameters.put("commandSpec", ""); } @@ -337,6 +340,37 @@ public Builder setDirectoriesToCopy(Set directoriesToCopy) { return addStringParameter("directoriesToCopy", directories.toString()); } + /** + * Sets the requirements file used for {@code pip install} and {@code pip download} at Docker + * build time. Defaults to {@code requirements.txt}. + * + * @param requirementsFile the requirements filename (e.g. "requirements_all.txt"). + * @return this {@link Builder}. + */ + public Builder setRequirementsFile(String requirementsFile) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(requirementsFile)); + return addStringParameter("requirementsFile", requirementsFile); + } + + /** + * Configures the Dockerfile to install a Python package via {@code pip install .} at build time + * and sets {@code FLEX_TEMPLATE_PYTHON_SETUP_FILE} so the Beam stager packages the source code + * for distribution to workers. The absolute path avoids issues with {@code os.chdir()}. + * + * @param setupFile the setup file name (e.g. "setup.py"). + * @return this {@link Builder}. + */ + public Builder setSetupFile(String setupFile) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(setupFile)); + String workDir = + (String) this.parameters.getOrDefault("workingDirectory", DEFAULT_WORKING_DIRECTORY); + addParameter( + "setupFileEnv", + "ENV FLEX_TEMPLATE_PYTHON_SETUP_FILE=\"" + workDir + "/" + setupFile + "\""); + addParameter("setupInstall", "RUN pip install --no-cache-dir ."); + return this; + } + /** * For XLANG templates, set the {@code DATAFLOW_JAVA_COMMAND_SPEC} env variable to the command * spec location on the image. diff --git a/plugins/core-plugin/src/main/resources/Dockerfile-template-python b/plugins/core-plugin/src/main/resources/Dockerfile-template-python index 8f09eb1e8f..5922e84814 100644 --- a/plugins/core-plugin/src/main/resources/Dockerfile-template-python +++ b/plugins/core-plugin/src/main/resources/Dockerfile-template-python @@ -3,17 +3,24 @@ FROM ${basePythonContainerImage} ARG WORKDIR=${workingDirectory} RUN mkdir -p $WORKDIR ${filesToCopy} +${directoriesToCopy} WORKDIR $WORKDIR -ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE=requirements.txt +# Do NOT use ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE here. That env var triggers +# the Beam stager to run pip download --platform manylinux2014_x86_64 for every package, +# which is broken for packages that no longer publish manylinux2014 wheels (e.g. numpy 2.4+). +# See https://github.com/pypa/pip/issues/10760 +# FLEX_TEMPLATE_PYTHON_SETUP_FILE (via setupFileEnv) IS needed to stage custom code to workers. ENV FLEX_TEMPLATE_PYTHON_PY_FILE=main.py +${setupFileEnv} -RUN if ! [ -f requirements.txt ] ; then >&2 echo "error: no requirements.txt file found" && exit 1 ; fi +RUN if ! [ -f ${requirementsFile} ] ; then >&2 echo "error: no ${requirementsFile} file found" && exit 1 ; fi # Set up custom PyPi repository, if applicable ${airlockConfig} -RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE -RUN pip download --require-hashes --no-cache-dir --dest /tmp/dataflow-requirements-cache -r $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE +RUN pip install -U --require-hashes -r ${requirementsFile} +${setupInstall} +RUN pip download --require-hashes --no-cache-dir --dest /tmp/dataflow-requirements-cache -r ${requirementsFile} -ENTRYPOINT ${entryPoint} \ No newline at end of file +ENTRYPOINT ${entryPoint} diff --git a/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java b/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java index b8e51a8189..1a2843cc48 100644 --- a/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java +++ b/plugins/core-plugin/src/test/java/com/google/cloud/teleport/plugin/DockerfileGeneratorTest.java @@ -58,8 +58,7 @@ public void testGeneratePythonDockerfileDefaults() throws IOException, TemplateE assertTrue(outputFile.exists()); String fileContents = Files.asCharSource(outputFile, StandardCharsets.UTF_8).read(); assertThat(fileContents).contains("FROM " + BASE_PYTHON_CONTAINER_IMAGE); - assertThat(fileContents) - .contains("RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE"); + assertThat(fileContents).contains("RUN pip install -U --require-hashes -r requirements.txt"); assertThat(fileContents) .contains(String.format("ENTRYPOINT [\"%s\"]", PYTHON_LAUNCHER_ENTRYPOINT)); } @@ -80,8 +79,7 @@ public void testGeneratePythonDockerfile() throws IOException, TemplateException assertTrue(outputFile.exists()); String fileContents = Files.asCharSource(outputFile, StandardCharsets.UTF_8).read(); assertThat(fileContents).contains("FROM a python container image"); - assertThat(fileContents) - .contains("RUN pip install -U -r --require-hashes $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE"); + assertThat(fileContents).contains("RUN pip install -U --require-hashes -r requirements.txt"); assertThat(fileContents).contains("COPY main.py requirements.txt $WORKDIR/"); assertThat(fileContents).contains("ENTRYPOINT [\"python/entry/point\"]"); } diff --git a/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java b/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java index 8b4e21c3a5..ff511a9489 100644 --- a/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java +++ b/plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java @@ -1008,7 +1008,7 @@ void prepareYamlDockerfile(TemplateDefinitions definition, String containerName) } List entryPoint = List.of(definition.getTemplateAnnotation().entryPoint()); - if (entryPoint.isEmpty()) { + if (entryPoint.isEmpty() || (entryPoint.size() == 1 && entryPoint.get(0).isEmpty())) { entryPoint = List.of(pythonTemplateLauncherEntryPoint); } @@ -1053,25 +1053,27 @@ private void stageFlexPythonTemplate( String dockerfilePath = dockerfileContainer + "/Dockerfile"; File dockerfile = new File(dockerfilePath); if (!dockerfile.exists()) { - List filesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy()); - if (filesToCopy.isEmpty()) { - filesToCopy = List.of("main.py", "requirements.txt"); + List allFilesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy()); + if (allFilesToCopy.isEmpty()) { + allFilesToCopy = List.of("main.py", "requirements.txt"); + } + + // Separate flat files from directories + List filesToCopy = new ArrayList<>(); + Set directoriesToCopy = new HashSet<>(); + for (String f : allFilesToCopy) { + File source = new File(dockerfileContainer + "/" + f); + if (source.isDirectory()) { + directoriesToCopy.add(f); + } else { + filesToCopy.add(f); + } } List entryPoint = List.of(definition.getTemplateAnnotation().entryPoint()); - if (entryPoint.isEmpty()) { + if (entryPoint.isEmpty() || (entryPoint.size() == 1 && entryPoint.get(0).isEmpty())) { entryPoint = List.of(pythonTemplateLauncherEntryPoint); } - // Copy in requirements.txt if present - File sourceRequirements = new File(outputClassesDirectory.getPath() + "/requirements.txt"); - File destRequirements = new File(dockerfileContainer + "/requirements.txt"); - if (sourceRequirements.exists()) { - Files.copy( - sourceRequirements.toPath(), - destRequirements.toPath(), - StandardCopyOption.REPLACE_EXISTING); - } - // Generate Dockerfile LOG.info("Generating dockerfile " + dockerfilePath); DockerfileGenerator.Builder dockerfileBuilder = @@ -1079,11 +1081,27 @@ private void stageFlexPythonTemplate( definition.getTemplateAnnotation().type(), beamVersion, containerName, - targetDirectory) + outputClassesDirectory) .setBasePythonContainerImage(basePythonContainerImage) .setFilesToCopy(filesToCopy) .setEntryPoint(entryPoint); + if (!directoriesToCopy.isEmpty()) { + dockerfileBuilder.setDirectoriesToCopy(directoriesToCopy); + } + + // Configure setup.py support if present + File setupFile = new File(dockerfileContainer + "/setup.py"); + if (setupFile.exists()) { + dockerfileBuilder.setSetupFile("setup.py"); + } + + // Use requirements_all.txt if present (full build lockfile, separate from worker deps) + File requirementsAll = new File(dockerfileContainer + "/requirements_all.txt"); + if (requirementsAll.exists()) { + dockerfileBuilder.setRequirementsFile("requirements_all.txt"); + } + // Set Airlock parameters if (internalMaven) { dockerfileBuilder diff --git a/pom.xml b/pom.xml index a35bfc36db..2780890927 100644 --- a/pom.xml +++ b/pom.xml @@ -615,6 +615,7 @@ **/KafkaToKafkaIT.java + **/BigQueryAnomalyDetectionIT.java ${direct-runner.tests} @@ -878,6 +879,7 @@ **/KafkaToKafkaIT.java + **/BigQueryAnomalyDetectionIT.java ${direct-runner.tests}, diff --git a/pr2-bqmonitor-source.txt b/pr2-bqmonitor-source.txt new file mode 100644 index 0000000000..9d066073c4 --- /dev/null +++ b/pr2-bqmonitor-source.txt @@ -0,0 +1,6862 @@ +diff --git a/python/default_base_bqmonitor_requirements.txt b/python/default_base_bqmonitor_requirements.txt +new file mode 100644 +index 000000000..abd9ea899 +--- /dev/null ++++ b/python/default_base_bqmonitor_requirements.txt +@@ -0,0 +1,3 @@ ++apache-beam[gcp]==2.71.0 ++google-cloud-bigquery-storage ++setuptools +diff --git a/python/generate_all_dependencies.sh b/python/generate_all_dependencies.sh +index 1e0035cdd..040be3973 100755 +--- a/python/generate_all_dependencies.sh ++++ b/python/generate_all_dependencies.sh +@@ -20,6 +20,7 @@ set -e + SCRIPTPATH=$(dirname "$0") + + sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/streaming-llm/base_requirements.txt $SCRIPTPATH/../python/src/main/python/streaming-llm/requirements.txt ++sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_bqmonitor_requirements.txt $SCRIPTPATH/../python/src/main/python/bigquery-anomaly-detection/requirements.txt + # Generate a base set of dependencies to use for any templates without special dependencies + mkdir -p $SCRIPTPATH/__build__/ + sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_python_requirements.txt $SCRIPTPATH/__build__/default_python_requirements.txt +diff --git a/python/pom.xml b/python/pom.xml +index af8a7bf57..56bff10c5 100644 +--- a/python/pom.xml ++++ b/python/pom.xml +@@ -83,6 +83,74 @@ + + + ++ ++ bqmonitorPythonTests ++ ++ false ++ ++ ++ ++ ++ org.codehaus.mojo ++ exec-maven-plugin ++ ${exec-maven-plugin.version} ++ ++ ++ bqmonitor-pip-install ++ test-compile ++ ++ exec ++ ++ ++ pip ++ ++ install ++ -r ++ src/test/python/bigquery-anomaly-detection/requirements-test.txt ++ ++ ++ ++ ++ bqmonitor-pip-install-pkg ++ test-compile ++ ++ exec ++ ++ ++ pip ++ ++ install ++ -e ++ src/main/python/bigquery-anomaly-detection ++ ++ ++ ++ ++ bqmonitor-python-test ++ test ++ ++ exec ++ ++ ++ python ++ ${project.basedir} ++ ++ -m ++ unittest ++ discover ++ -s ++ src/test/python/bigquery-anomaly-detection ++ -p ++ *_test.py ++ -v ++ ++ ++ ++ ++ ++ ++ ++ + + templatesValidate + +diff --git a/python/src/main/python/bigquery-anomaly-detection/main.py b/python/src/main/python/bigquery-anomaly-detection/main.py +new file mode 100644 +index 000000000..afd7bf028 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/main.py +@@ -0,0 +1,25 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Flex Template entry point for bqmonitor.""" ++ ++import logging ++ ++from bqmonitor.pipeline import run ++ ++if __name__ == '__main__': ++ logging.getLogger().setLevel(logging.INFO) ++ run() +diff --git a/python/src/main/python/bigquery-anomaly-detection/pyproject.toml b/python/src/main/python/bigquery-anomaly-detection/pyproject.toml +new file mode 100644 +index 000000000..c36dd5d36 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/pyproject.toml +@@ -0,0 +1,13 @@ ++[build-system] ++requires = ["setuptools>=64", "wheel"] ++build-backend = "setuptools.build_meta" ++ ++[project] ++name = "bqmonitor" ++version = "0.1.0" ++description = "BigQuery anomaly monitoring pipeline (Dataflow Flex Template)" ++requires-python = ">=3.11" ++dependencies = [ ++ "apache-beam[gcp]==2.71.0", ++ "google-cloud-bigquery-storage", ++] +diff --git a/python/src/main/python/bigquery-anomaly-detection/requirements.txt b/python/src/main/python/bigquery-anomaly-detection/requirements.txt +new file mode 100644 +index 000000000..ecae6a536 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/requirements.txt +@@ -0,0 +1,2579 @@ ++# Copyright 2025 Google Inc. All Rights Reserved. ++ ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++ ++# http://www.apache.org/licenses/LICENSE-2.0 ++ ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++# Autogenerated requirements file for Apache Beam container image. ++# From the templates base directory to update, ++# run: sh python/generate_all_dependencies.sh ++# Do not edit manually, adjust the base requirements file, and regenerate the list. ++ ++# See [maintainers-guide](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/contributor-docs/maintainers-guide.md#validating-and-upgrading-beam-versions) for more information. ++ ++# ++# This file is autogenerated by pip-compile with Python 3.11 ++# by the following command: ++# ++# pip-compile --allow-unsafe --generate-hashes --output-file=python/../python/src/main/python/bigquery-anomaly-detection/requirements.txt python/default_base_bqmonitor_requirements.txt ++# ++aiofiles==25.1.0 \ ++ --hash=sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2 \ ++ --hash=sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695 ++ # via cloud-sql-python-connector ++aiohappyeyeballs==2.6.1 \ ++ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ ++ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 ++ # via aiohttp ++aiohttp==3.13.3 \ ++ --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf \ ++ --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c \ ++ --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c \ ++ --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 \ ++ --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f \ ++ --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 \ ++ --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 \ ++ --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf \ ++ --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 \ ++ --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 \ ++ --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 \ ++ --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 \ ++ --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d \ ++ --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea \ ++ --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 \ ++ --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 \ ++ --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 \ ++ --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 \ ++ --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 \ ++ --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 \ ++ --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a \ ++ --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e \ ++ --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 \ ++ --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 \ ++ --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 \ ++ --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd \ ++ --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a \ ++ --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 \ ++ --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 \ ++ --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 \ ++ --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f \ ++ --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 \ ++ --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 \ ++ --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 \ ++ --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 \ ++ --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce \ ++ --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 \ ++ --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 \ ++ --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 \ ++ --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 \ ++ --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b \ ++ --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 \ ++ --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c \ ++ --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 \ ++ --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 \ ++ --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f \ ++ --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b \ ++ --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e \ ++ --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 \ ++ --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 \ ++ --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 \ ++ --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d \ ++ --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 \ ++ --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 \ ++ --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce \ ++ --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 \ ++ --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 \ ++ --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c \ ++ --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf \ ++ --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 \ ++ --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 \ ++ --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e \ ++ --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 \ ++ --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a \ ++ --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 \ ++ --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a \ ++ --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 \ ++ --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 \ ++ --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba \ ++ --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df \ ++ --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 \ ++ --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 \ ++ --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 \ ++ --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 \ ++ --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc \ ++ --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 \ ++ --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 \ ++ --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 \ ++ --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 \ ++ --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 \ ++ --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 \ ++ --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 \ ++ --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 \ ++ --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 \ ++ --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 \ ++ --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc \ ++ --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 \ ++ --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 \ ++ --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 \ ++ --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 \ ++ --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 \ ++ --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 \ ++ --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c \ ++ --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 \ ++ --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 \ ++ --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a \ ++ --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 \ ++ --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 \ ++ --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 \ ++ --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d \ ++ --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 \ ++ --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d \ ++ --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f \ ++ --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f \ ++ --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 \ ++ --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 \ ++ --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 \ ++ --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 \ ++ --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af \ ++ --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 \ ++ --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 \ ++ --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 \ ++ --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c \ ++ --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 \ ++ --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 \ ++ --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 \ ++ --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 \ ++ --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 \ ++ --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa \ ++ --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 ++ # via cloud-sql-python-connector ++aiosignal==1.4.0 \ ++ --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ ++ --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 ++ # via aiohttp ++annotated-types==0.7.0 \ ++ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ ++ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 ++ # via pydantic ++anyio==4.12.1 \ ++ --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ ++ --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c ++ # via ++ # google-genai ++ # httpx ++apache-beam[gcp]==2.71.0 \ ++ --hash=sha256:044841032ef190a7ad69a9d4ca4b23c104a310d08c47a1f5faefcf830c9e5520 \ ++ --hash=sha256:192b00d13de8eb06241c5332ecef7a9a947758e2103b07d6726848ba9f0b5a49 \ ++ --hash=sha256:1cdaf7e502da67f674ecf8dd8cec21252bd1b2678a5d18b873a45635cf0e7cec \ ++ --hash=sha256:28c6eeb05b688dcc503fce84075fcd03a73bbd9e449e70521f2efb47a932bcea \ ++ --hash=sha256:317f5495c3266b9146263dbb881110b56b015fbc7e2f1e27eb9932b2bf28a94c \ ++ --hash=sha256:3705d824d462aee4bf162318eb0ef1ca767064e73aa4f1ba14d741cc12c19143 \ ++ --hash=sha256:43ed7ae3dbecf67af2ad412b86d160fc6177d19fc6e59ed18aee4a84355858db \ ++ --hash=sha256:515064493c478e92a87618f46c8b8c2143ce244317db683dc3d824fda37b0db5 \ ++ --hash=sha256:5ca7fca47ae39b5e6497c39bca303d11c200fdfae6b352e5e481a59a9b886f75 \ ++ --hash=sha256:78c2f8e88014555984a7a21bcb63479e135b958428d178d45699a4154ae84634 \ ++ --hash=sha256:78e3e913275bd1c1aac1ecc90af78fb65915908671b6e39d60a3a31de3438782 \ ++ --hash=sha256:81766907e53a5feddb2d1b5553c6f1154ff7cae67e548b4c2726e299334572bf \ ++ --hash=sha256:8189d2e1d314a7dc8f3456bae4c7641637d302490e1af93db3aa6ba45d716b70 \ ++ --hash=sha256:83be2fce3726529f221c8d99f844f64d68494b2bad438852f96f02f2c0e8cac8 \ ++ --hash=sha256:8e1cbc386cf8c0d740b3b2847cb7c99481672ed036b57c11eb2f41d049800b40 \ ++ --hash=sha256:a11147b82260d69b19021b32a65da044d38f65195ec2a66460ccad80649106b5 \ ++ --hash=sha256:a14fb6972de7113dfbe6bba967de1a3a5c60228a96b96eb32a675762f83d659b \ ++ --hash=sha256:a358a7e689e1acb903ec5f545ed22b674fb6cbb17424518630412cba3a627937 \ ++ --hash=sha256:a7967a1d75daec31e9d03705304ad4e7e5bcad266dd5e8bad98a68e76ebb368f \ ++ --hash=sha256:af5a9acf850b8430440f8e6f687650c252dd7d0b929fbef2d84ce79087f6bb6b \ ++ --hash=sha256:b3acb72a5afdc15abe696e37915cbce91d7a0672fda2658c2185d8ea4684d4e3 \ ++ --hash=sha256:c015aa7ee75cabc58277b19317429fc3ed08752173d6750b2212260190505c7f \ ++ --hash=sha256:d4a3b4008ca3966f426a8580535e2227387518a2d62c3928c4e3d5a6ca23dd8a \ ++ --hash=sha256:de890d820ae365eddcbe522e61816a967ab9d5be501fb56435e0d8a8c571408e \ ++ --hash=sha256:e06fb7fd4f5aa9d16bb8d8d30d9c24fc255cfc9be510188bfab0b11f398cc515 ++ # via -r python/default_base_bqmonitor_requirements.txt ++asn1crypto==1.5.1 \ ++ --hash=sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c \ ++ --hash=sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67 ++ # via scramp ++attrs==25.4.0 \ ++ --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ ++ --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 ++ # via aiohttp ++backports-tarfile==1.2.0 \ ++ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ ++ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 ++ # via jaraco-context ++beartype==0.22.9 \ ++ --hash=sha256:8f82b54aa723a2848a56008d18875f91c1db02c32ef6a62319a002e3e25a975f \ ++ --hash=sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2 ++ # via apache-beam ++betterproto==2.0.0b7 \ ++ --hash=sha256:1b1458ca5278d519bcd62556a4c236f998a91d503f0f71c67b0b954747052af2 \ ++ --hash=sha256:401ab8055e2f814e77b9c88a74d0e1ae3d1e8a969cced6aeb1b59f71ad63fbd2 ++ # via envoy-data-plane ++cachetools==6.2.6 \ ++ --hash=sha256:16c33e1f276b9a9c0b49ab5782d901e3ad3de0dd6da9bf9bcd29ac5672f2f9e6 \ ++ --hash=sha256:8c9717235b3c651603fff0076db52d6acbfd1b338b8ed50256092f7ce9c85bda ++ # via apache-beam ++certifi==2026.2.25 \ ++ --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ ++ --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 ++ # via ++ # httpcore ++ # httpx ++ # requests ++cffi==2.0.0 \ ++ --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ ++ --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ ++ --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ ++ --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ ++ --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ ++ --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ ++ --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ ++ --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ ++ --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ ++ --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ ++ --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ ++ --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ ++ --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ ++ --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ ++ --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ ++ --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ ++ --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ ++ --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ ++ --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ ++ --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ ++ --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ ++ --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ ++ --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ ++ --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ ++ --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ ++ --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ ++ --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ ++ --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ ++ --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ ++ --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ ++ --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ ++ --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ ++ --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ ++ --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ ++ --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ ++ --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ ++ --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ ++ --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ ++ --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ ++ --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ ++ --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ ++ --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ ++ --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ ++ --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ ++ --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ ++ --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ ++ --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ ++ --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ ++ --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ ++ --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ ++ --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ ++ --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ ++ --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ ++ --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ ++ --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ ++ --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ ++ --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ ++ --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ ++ --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ ++ --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ ++ --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ ++ --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ ++ --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ ++ --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ ++ --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ ++ --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ ++ --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ ++ --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ ++ --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ ++ --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ ++ --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ ++ --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ ++ --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ ++ --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ ++ --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ ++ --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ ++ --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ ++ --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ ++ --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ ++ --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ ++ --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ ++ --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ ++ --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ ++ --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf ++ # via cryptography ++charset-normalizer==3.4.5 \ ++ --hash=sha256:014837af6fabf57121b6254fa8ade10dceabc3528b27b721a64bbc7b8b1d4eb4 \ ++ --hash=sha256:01a1ed54b953303ca7e310fafe0fe347aab348bd81834a0bcd602eb538f89d66 \ ++ --hash=sha256:0294916d6ccf2d069727d65973c3a1ca477d68708db25fd758dd28b0827cff54 \ ++ --hash=sha256:02a9d1b01c1e12c27883b0c9349e0bcd9ae92e727ff1a277207e1a262b1cbf05 \ ++ --hash=sha256:036c079aa08a6a592b82487f97c60b439428320ed1b2ea0b3912e99d30c77765 \ ++ --hash=sha256:039215608ac7b358c4da0191d10fc76868567fbf276d54c14721bdedeb6de064 \ ++ --hash=sha256:0625665e4ebdddb553ab185de5db7054393af8879fb0c87bd5690d14379d6819 \ ++ --hash=sha256:0a45e504f5e1be0bd385935a8e1507c442349ca36f511a47057a71c9d1d6ea9e \ ++ --hash=sha256:0b362bcd27819f9c07cbf23db4e0e8cd4b44c5ecd900c2ff907b2b92274a7412 \ ++ --hash=sha256:0c300cefd9b0970381a46394902cd18eaf2aa00163f999590ace991989dcd0fc \ ++ --hash=sha256:1088345bcc93c58d8d8f3d783eca4a6e7a7752bbff26c3eee7e73c597c191c2e \ ++ --hash=sha256:10b473fc8dca1c3ad8559985794815f06ca3fc71942c969129070f2c3cdf7281 \ ++ --hash=sha256:131716d6786ad5e3dc542f5cc6f397ba3339dc0fb87f87ac30e550e8987756af \ ++ --hash=sha256:14498a429321de554b140013142abe7608f9d8ccc04d7baf2ad60498374aefa2 \ ++ --hash=sha256:149ec69866c3d6c2fb6f758dbc014ecb09f30b35a5ca90b6a8a2d4e54e18fdfe \ ++ --hash=sha256:165c7b21d19365464e8f70e5ce5e12524c58b48c78c1f5a57524603c1ab003f8 \ ++ --hash=sha256:1827734a5b308b65ac54e86a618de66f935a4f63a8a462ff1e19a6788d6c2262 \ ++ --hash=sha256:19092dde50335accf365cce21998a1c6dd8eafd42c7b226eb54b2747cdce2fac \ ++ --hash=sha256:1a374cc0b88aa710e8865dc1bd6edb3743c59f27830f0293ab101e4cf3ce9f85 \ ++ --hash=sha256:1d1401945cb77787dbd3af2446ff2d75912327c4c3a1526ab7955ecf8600687c \ ++ --hash=sha256:1f2da5cbb9becfcd607757a169e38fb82aa5fd86fae6653dea716e7b613fe2cf \ ++ --hash=sha256:259cd1ca995ad525f638e131dbcc2353a586564c038fc548a3fe450a91882139 \ ++ --hash=sha256:2820a98460c83663dd8ec015d9ddfd1e4879f12e06bb7d0500f044fb477d2770 \ ++ --hash=sha256:28269983f25a4da0425743d0d257a2d6921ea7d9b83599d4039486ec5b9f911d \ ++ --hash=sha256:2b970382e4a36bed897c19f310f31d7d13489c11b4f468ddfba42d41cddfb918 \ ++ --hash=sha256:2da4eedcb6338e2321e831a0165759c0c620e37f8cd044a263ff67493be8ffb3 \ ++ --hash=sha256:30987f4a8ed169983f93e1be8ffeea5214a779e27ed0b059835c7afe96550ad7 \ ++ --hash=sha256:30a2b1a48478c3428d047ed9690d57c23038dac838a87ad624c85c0a78ebeb39 \ ++ --hash=sha256:340810d34ef83af92148e96e3e44cb2d3f910d2bf95e5618a5c467d9f102231d \ ++ --hash=sha256:3f64c6bf8f32f9133b668c7f7a7cbdbc453412bc95ecdbd157f3b1e377a92990 \ ++ --hash=sha256:4167a621a9a1a986c73777dbc15d4b5eac8ac5c10393374109a343d4013ec765 \ ++ --hash=sha256:4354e401eb6dab9aed3c7b4030514328a6c748d05e1c3e19175008ca7de84fb1 \ ++ --hash=sha256:4481e6da1830c8a1cc0b746b47f603b653dadb690bcd851d039ffaefe70533aa \ ++ --hash=sha256:4b8551b6e6531e156db71193771c93bda78ffc4d1e6372517fe58ad3b91e4659 \ ++ --hash=sha256:4cd966c2559f501c6fd69294d082c2934c8dd4719deb32c22961a5ac6db0df1d \ ++ --hash=sha256:50bcbca6603c06a1dcc7b056ed45c37715fb5d2768feb3bcd37d2313c587a5b9 \ ++ --hash=sha256:530beedcec9b6e027e7a4b6ce26eed36678aa39e17da85e6e03d7bd9e8e9d7c9 \ ++ --hash=sha256:568e3c34b58422075a1b49575a6abc616d9751b4d61b23f712e12ebb78fe47b2 \ ++ --hash=sha256:573ef5814c4b7c0d59a7710aa920eaaaef383bd71626aa420fba27b5cab92e8d \ ++ --hash=sha256:58ad8270cfa5d4bef1bc85bd387217e14ff154d6630e976c6f56f9a040757475 \ ++ --hash=sha256:597d10dec876923e5c59e48dbd366e852eacb2b806029491d307daea6b917d7c \ ++ --hash=sha256:5bcb3227c3d9aaf73eaaab1db7ccd80a8995c509ee9941e2aae060ca6e4e5d81 \ ++ --hash=sha256:5cffde4032a197bd3b42fd0b9509ec60fb70918d6970e4cc773f20fc9180ca67 \ ++ --hash=sha256:5fea359734b140d0d6741189fea5478c6091b54ffc69d7ce119e0a05637d8c99 \ ++ --hash=sha256:60d68e820af339df4ae8358c7a2e7596badeb61e544438e489035f9fbf3246a5 \ ++ --hash=sha256:610f72c0ee565dfb8ae1241b666119582fdbfe7c0975c175be719f940e110694 \ ++ --hash=sha256:65a126fb4b070d05340a84fc709dd9e7c75d9b063b610ece8a60197a291d0adf \ ++ --hash=sha256:65b3c403a5b6b8034b655e7385de4f72b7b244869a22b32d4030b99a60593eca \ ++ --hash=sha256:66dee73039277eb35380d1b82cccc69cc82b13a66f9f4a18da32d573acf02b7c \ ++ --hash=sha256:708c7acde173eedd4bfa4028484426ba689d2103b28588c513b9db2cd5ecde9c \ ++ --hash=sha256:728c6a963dfab66ef865f49286e45239384249672cd598576765acc2a640a636 \ ++ --hash=sha256:754f96058e61a5e22e91483f823e07df16416ce76afa4ebf306f8e1d1296d43f \ ++ --hash=sha256:75dfd1afe0b1647449e852f4fb428195a7ed0588947218f7ba929f6538487f02 \ ++ --hash=sha256:75ee9c1cce2911581a70a3c0919d8bccf5b1cbc9b0e5171400ec736b4b569497 \ ++ --hash=sha256:76a9d0de4d0eab387822e7b35d8f89367dd237c72e82ab42b9f7bf5e15ada00f \ ++ --hash=sha256:77be992288f720306ab4108fe5c74797de327f3248368dfc7e1a916d6ed9e5a2 \ ++ --hash=sha256:7ad83b8f9379176c841f8865884f3514d905bcd2a9a3b210eaa446e7d2223e4d \ ++ --hash=sha256:8197abe5ca1ffb7d91e78360f915eef5addff270f8a71c1fc5be24a56f3e4873 \ ++ --hash=sha256:82cc7c2ad42faec8b574351f8bc2a0c049043893853317bd9bb309f5aba6cb5a \ ++ --hash=sha256:8a28afb04baa55abf26df544e3e5c6534245d3daa5178bc4a8eeb48202060d0e \ ++ --hash=sha256:8b78d8a609a4b82c273257ee9d631ded7fac0d875bdcdccc109f3ee8328cfcb1 \ ++ --hash=sha256:8ce11cd4d62d11166f2b441e30ace226c19a3899a7cf0796f668fba49a9fb123 \ ++ --hash=sha256:8fff79bf5978c693c9b1a4d71e4a94fddfb5fe744eb062a318e15f4a2f63a550 \ ++ --hash=sha256:92263f7eca2f4af326cd20de8d16728d2602f7cfea02e790dcde9d83c365d7cc \ ++ --hash=sha256:93b3b2cc5cf1b8743660ce77a4f45f3f6d1172068207c1defc779a36eea6bb36 \ ++ --hash=sha256:95adae7b6c42a6c5b5b559b1a99149f090a57128155daeea91732c8d970d8644 \ ++ --hash=sha256:97ab7787092eb9b50fb47fa04f24c75b768a606af1bcba1957f07f128a7219e4 \ ++ --hash=sha256:9db5e3fcdcee89a78c04dffb3fe33c79f77bd741a624946db2591c81b2fc85b0 \ ++ --hash=sha256:a118e2e0b5ae6b0120d5efa5f866e58f2bb826067a646431da4d6a2bdae7950e \ ++ --hash=sha256:a2aecdb364b8a1802afdc7f9327d55dad5366bc97d8502d0f5854e50712dbc5f \ ++ --hash=sha256:a66aa5022bf81ab4b1bebfb009db4fd68e0c6d4307a1ce5ef6a26e5878dfc9e4 \ ++ --hash=sha256:a68766a3c58fde7f9aaa22b3786276f62ab2f594efb02d0a1421b6282e852e98 \ ++ --hash=sha256:aa2f963b4da26daf46231d9b9e0e2c9408a751f8f0d0f44d2de56d3caf51d294 \ ++ --hash=sha256:aa92ec1102eaff840ccd1021478af176a831f1bccb08e526ce844b7ddda85c22 \ ++ --hash=sha256:ac59c15e3f1465f722607800c68713f9fbc2f672b9eb649fe831da4019ae9b23 \ ++ --hash=sha256:ae8b03427410731469c4033934cf473426faff3e04b69d2dfb64a4281a3719f8 \ ++ --hash=sha256:afca7f78067dd27c2b848f1b234623d26b87529296c6c5652168cc1954f2f3b2 \ ++ --hash=sha256:b2d37d78297b39a9eb9eb92c0f6df98c706467282055419df141389b23f93362 \ ++ --hash=sha256:b3e71afc578b98512bfe7bdb822dd6bc57d4b0093b4b6e5487c1e96ad4ace242 \ ++ --hash=sha256:ba20bdf69bd127f66d0174d6f2a93e69045e0b4036dc1ca78e091bcc765830c4 \ ++ --hash=sha256:c108f8619e504140569ee7de3f97d234f0fbae338a7f9f360455071ef9855a95 \ ++ --hash=sha256:c23eb3263356d94858655b3e63f85ac5d50970c6e8febcdde7830209139cc37d \ ++ --hash=sha256:c5af897b45fa606b12464ccbe0014bbf8c09191e0a66aab6aa9d5cf6e77e0c94 \ ++ --hash=sha256:c7a80a9242963416bd81f99349d5f3fce1843c303bd404f204918b6d75a75fd6 \ ++ --hash=sha256:c7e84e0c0005e3bdc1a9211cd4e62c78ba80bc37b2365ef4410cd2007a9047f2 \ ++ --hash=sha256:cace89841c0599d736d3d74a27bc5821288bb47c5441923277afc6059d7fbcb4 \ ++ --hash=sha256:cd2d0f0ec9aa977a27731a3209ebbcacebebaf41f902bd453a928bfd281cf7f8 \ ++ --hash=sha256:d01de5e768328646e6a3fa9e562706f8f6641708c115c62588aef2b941a4f88e \ ++ --hash=sha256:d1028de43596a315e2720a9849ee79007ab742c06ad8b45a50db8cdb7ed4a82a \ ++ --hash=sha256:d27ce22ec453564770d29d03a9506d449efbb9fa13c00842262b2f6801c48cce \ ++ --hash=sha256:d29dd9c016f2078b43d0c357511e87eee5b05108f3dd603423cb389b89813969 \ ++ --hash=sha256:d31f0d1671e1534e395f9eb84a68e0fb670e1edb1fe819a9d7f564ae3bc4e53f \ ++ --hash=sha256:d4eb8ac7469b2a5d64b5b8c04f84d8bf3ad340f4514b98523805cbf46e3b3923 \ ++ --hash=sha256:d5e52d127045d6ae01a1e821acfad2f3a1866c54d0e837828538fabe8d9d1bd6 \ ++ --hash=sha256:d77f97e515688bd615c1d1f795d540f32542d514242067adcb8ef532504cb9ee \ ++ --hash=sha256:d8ed79b8f6372ca4254955005830fd61c1ccdd8c0fac6603e2c145c61dd95db6 \ ++ --hash=sha256:dc57a0baa3eeedd99fafaef7511b5a6ef4581494e8168ee086031744e2679467 \ ++ --hash=sha256:e09f671a54ce70b79a1fc1dc6da3072b7ef7251fadb894ed92d9aa8218465a5f \ ++ --hash=sha256:e22d1059b951e7ae7c20ef6b06afd10fb95e3c41bf3c4fbc874dba113321c193 \ ++ --hash=sha256:e37bd100d2c5d3ba35db9c7c5ba5a9228cbcffe5c4778dc824b164e5257813d7 \ ++ --hash=sha256:e51ae7d81c825761d941962450f50d041db028b7278e7b08930b4541b3e45cb9 \ ++ --hash=sha256:e545b51da9f9af5c67815ca0eb40676c0f016d0b0381c86f20451e35696c5f95 \ ++ --hash=sha256:e6302ca4ae283deb0af68d2fbf467474b8b6aedcd3dab4db187e07f94c109763 \ ++ --hash=sha256:e71bbb595973622b817c042bd943c3f3667e9c9983ce3d205f973f486fec98a7 \ ++ --hash=sha256:ec56a2266f32bc06ed3c3e2a8f58417ce02f7e0356edc89786e52db13c593c98 \ ++ --hash=sha256:ed1a9a204f317ef879b32f9af507d47e49cd5e7f8e8d5d96358c98373314fc60 \ ++ --hash=sha256:ed97c282ee4f994ef814042423a529df9497e3c666dca19be1d4cd1129dc7ade \ ++ --hash=sha256:ed98364e1c262cf5f9363c3eca8c2df37024f52a8fa1180a3610014f26eac51c \ ++ --hash=sha256:ee57b926940ba00bca7ba7041e665cc956e55ef482f851b9b65acb20d867e7a2 \ ++ --hash=sha256:f1d725b754e967e648046f00c4facc42d414840f5ccc670c5670f59f83693e4f \ ++ --hash=sha256:f8102ae93c0bc863b1d41ea0f4499c20a83229f52ed870850892df555187154a \ ++ --hash=sha256:fc1c64934b8faf7584924143eb9db4770bbdb16659626e1a1a4d9efbcb68d947 \ ++ --hash=sha256:ff95a9283de8a457e6b12989de3f9f5193430f375d64297d323a615ea52cbdb3 ++ # via requests ++cloud-sql-python-connector==1.20.0 \ ++ --hash=sha256:aa7c30631c5f455d14d561d7b0b414a97652a1b582a301f5570ba2cea2aa9105 \ ++ --hash=sha256:fdd96153b950040b0252453115604c142922b72cf3636146165a648ac5f6fc30 ++ # via apache-beam ++cryptography==46.0.5 \ ++ --hash=sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72 \ ++ --hash=sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235 \ ++ --hash=sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9 \ ++ --hash=sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356 \ ++ --hash=sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257 \ ++ --hash=sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad \ ++ --hash=sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4 \ ++ --hash=sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c \ ++ --hash=sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614 \ ++ --hash=sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed \ ++ --hash=sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31 \ ++ --hash=sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229 \ ++ --hash=sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0 \ ++ --hash=sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731 \ ++ --hash=sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b \ ++ --hash=sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4 \ ++ --hash=sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4 \ ++ --hash=sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263 \ ++ --hash=sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595 \ ++ --hash=sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1 \ ++ --hash=sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678 \ ++ --hash=sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48 \ ++ --hash=sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76 \ ++ --hash=sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0 \ ++ --hash=sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18 \ ++ --hash=sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d \ ++ --hash=sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d \ ++ --hash=sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1 \ ++ --hash=sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981 \ ++ --hash=sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7 \ ++ --hash=sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82 \ ++ --hash=sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2 \ ++ --hash=sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4 \ ++ --hash=sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663 \ ++ --hash=sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c \ ++ --hash=sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d \ ++ --hash=sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a \ ++ --hash=sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a \ ++ --hash=sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d \ ++ --hash=sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b \ ++ --hash=sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a \ ++ --hash=sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826 \ ++ --hash=sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee \ ++ --hash=sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9 \ ++ --hash=sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648 \ ++ --hash=sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da \ ++ --hash=sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2 \ ++ --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2 \ ++ --hash=sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87 ++ # via ++ # apache-beam ++ # cloud-sql-python-connector ++ # google-auth ++ # secretstorage ++distro==1.9.0 \ ++ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ ++ --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 ++ # via google-genai ++dnspython==2.8.0 \ ++ --hash=sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af \ ++ --hash=sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f ++ # via ++ # cloud-sql-python-connector ++ # pymongo ++docstring-parser==0.17.0 \ ++ --hash=sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912 \ ++ --hash=sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708 ++ # via google-cloud-aiplatform ++envoy-data-plane==0.2.6 \ ++ --hash=sha256:6341768b9cf5d6268baced4d2e8b3429f98664fbbe8958dae69ee25316ae869a \ ++ --hash=sha256:d1541c8cd00677886a2f93696edf9e3589cd4ac680defc66b3013ffb082f274c ++ # via apache-beam ++fastavro==1.12.1 \ ++ --hash=sha256:00650ca533907361edda22e6ffe8cf87ab2091c5d8aee5c8000b0f2dcdda7ed3 \ ++ --hash=sha256:02281432dcb11c78b3280da996eff61ee0eff39c5de06c6e0fbf19275093e6d4 \ ++ --hash=sha256:0714b285160fcd515eb0455540f40dd6dac93bdeacdb03f24e8eac3d8aa51f8d \ ++ --hash=sha256:089e155c0c76e0d418d7e79144ce000524dd345eab3bc1e9c5ae69d500f71b14 \ ++ --hash=sha256:120aaf82ac19d60a1016afe410935fe94728752d9c2d684e267e5b7f0e70f6d9 \ ++ --hash=sha256:123fb221df3164abd93f2d042c82f538a1d5a43ce41375f12c91ce1355a9141e \ ++ --hash=sha256:1f55eef18c41d4476bd32a82ed5dd86aabc3f614e1b66bdb09ffa291612e1670 \ ++ --hash=sha256:1f81011d54dd47b12437b51dd93a70a9aa17b61307abf26542fc3c13efbc6c51 \ ++ --hash=sha256:2de72d786eb38be6b16d556b27232b1bf1b2797ea09599507938cdb7a9fe3e7c \ ++ --hash=sha256:2f285be49e45bc047ab2f6bed040bb349da85db3f3c87880e4b92595ea093b2b \ ++ --hash=sha256:3100ad643e7fa658469a2a2db229981c1a000ff16b8037c0b58ce3ec4d2107e8 \ ++ --hash=sha256:3616e2f0e1c9265e92954fa099db79c6e7817356d3ff34f4bcc92699ae99697c \ ++ --hash=sha256:3b1921ac35f3d89090a5816b626cf46e67dbecf3f054131f84d56b4e70496f45 \ ++ --hash=sha256:4128978b930aaf930332db4b3acc290783183f3be06a241ae4a482f3ed8ce892 \ ++ --hash=sha256:43ded16b3f4a9f1a42f5970c2aa618acb23ea59c4fcaa06680bdf470b255e5a8 \ ++ --hash=sha256:44cbff7518901c91a82aab476fcab13d102e4999499df219d481b9e15f61af34 \ ++ --hash=sha256:469fecb25cba07f2e1bfa4c8d008477cd6b5b34a59d48715e1b1a73f6160097d \ ++ --hash=sha256:509818cb24b98a804fc80be9c5fed90f660310ae3d59382fc811bfa187122167 \ ++ --hash=sha256:5217f773492bac43dae15ff2931432bce2d7a80be7039685a78d3fab7df910bd \ ++ --hash=sha256:546ffffda6610fca672f0ed41149808e106d8272bb246aa7539fa8bb6f117f17 \ ++ --hash=sha256:5aa777b8ee595b50aa084104cd70670bf25a7bbb9fd8bb5d07524b0785ee1699 \ ++ --hash=sha256:632a4e3ff223f834ddb746baae0cc7cee1068eb12c32e4d982c2fee8a5b483d0 \ ++ --hash=sha256:64961ab15b74b7c168717bbece5660e0f3d457837c3cc9d9145181d011199fa7 \ ++ --hash=sha256:6b632b713bc5d03928a87d811fa4a11d5f25cd43e79c161e291c7d3f7aa740fd \ ++ --hash=sha256:780476c23175d2ae457c52f45b9ffa9d504593499a36cd3c1929662bf5b7b14b \ ++ --hash=sha256:78df838351e4dff9edd10a1c41d1324131ffecbadefb9c297d612ef5363c049a \ ++ --hash=sha256:792356d320f6e757e89f7ac9c22f481e546c886454a6709247f43c0dd7058004 \ ++ --hash=sha256:81563e1f93570e6565487cdb01ba241a36a00e58cff9c5a0614af819d1155d8f \ ++ --hash=sha256:83e6caf4e7a8717d932a3b1ff31595ad169289bbe1128a216be070d3a8391671 \ ++ --hash=sha256:9090f0dee63fe022ee9cc5147483366cc4171c821644c22da020d6b48f576b4f \ ++ --hash=sha256:9445da127751ba65975d8e4bdabf36bfcfdad70fc35b2d988e3950cce0ec0e7c \ ++ --hash=sha256:a275e48df0b1701bb764b18a8a21900b24cf882263cb03d35ecdba636bbc830b \ ++ --hash=sha256:a38607444281619eda3a9c1be9f5397634012d1b237142eee1540e810b30ac8b \ ++ --hash=sha256:a7d840ccd9aacada3ddc80fbcc4ea079b658107fe62e9d289a0de9d54e95d366 \ ++ --hash=sha256:a8bc2dcec5843d499f2489bfe0747999108f78c5b29295d877379f1972a3d41a \ ++ --hash=sha256:ac76d6d95f909c72ee70d314b460b7e711d928845771531d823eb96a10952d26 \ ++ --hash=sha256:b6a3462934b20a74f9ece1daa49c2e4e749bd9a35fa2657b53bf62898fba80f5 \ ++ --hash=sha256:b81fc04e85dfccf7c028e0580c606e33aa8472370b767ef058aae2c674a90746 \ ++ --hash=sha256:b91a0fe5a173679a6c02d53ca22dcaad0a2c726b74507e0c1c2e71a7c3f79ef9 \ ++ --hash=sha256:bec207360f76f0b3de540758a297193c5390e8e081c43c3317f610b1414d8c8f \ ++ --hash=sha256:c0390bfe4a9f8056a75ac6785fbbff8f5e317f5356481d2e29ec980877d2314b \ ++ --hash=sha256:c3d67c47f177e486640404a56f2f50b165fe892cc343ac3a34673b80cc7f1dd6 \ ++ --hash=sha256:cb0337b42fd3c047fcf0e9b7597bd6ad25868de719f29da81eabb6343f08d399 \ ++ --hash=sha256:d71c8aa841ef65cfab709a22bb887955f42934bced3ddb571e98fdbdade4c609 \ ++ --hash=sha256:eaa7ab3769beadcebb60f0539054c7755f63bd9cf7666e2c15e615ab605f89a8 \ ++ --hash=sha256:ed924233272719b5d5a6a0b4d80ef3345fc7e84fc7a382b6232192a9112d38a6 ++ # via apache-beam ++fasteners==0.20 \ ++ --hash=sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8 \ ++ --hash=sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7 ++ # via ++ # apache-beam ++ # google-apitools ++frozenlist==1.8.0 \ ++ --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ ++ --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ ++ --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ ++ --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ ++ --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ ++ --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ ++ --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ ++ --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ ++ --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ ++ --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ ++ --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ ++ --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ ++ --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ ++ --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ ++ --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ ++ --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ ++ --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ ++ --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ ++ --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ ++ --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ ++ --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ ++ --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ ++ --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ ++ --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ ++ --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ ++ --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ ++ --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ ++ --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ ++ --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ ++ --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ ++ --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ ++ --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ ++ --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ ++ --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ ++ --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ ++ --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ ++ --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ ++ --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ ++ --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ ++ --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ ++ --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ ++ --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ ++ --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ ++ --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ ++ --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ ++ --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ ++ --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ ++ --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ ++ --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ ++ --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ ++ --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ ++ --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ ++ --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ ++ --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ ++ --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ ++ --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ ++ --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ ++ --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ ++ --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ ++ --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ ++ --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ ++ --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ ++ --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ ++ --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ ++ --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ ++ --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ ++ --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ ++ --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ ++ --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ ++ --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ ++ --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ ++ --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ ++ --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ ++ --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ ++ --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ ++ --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ ++ --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ ++ --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ ++ --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ ++ --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ ++ --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ ++ --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ ++ --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ ++ --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ ++ --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ ++ --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ ++ --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ ++ --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ ++ --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ ++ --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ ++ --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ ++ --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ ++ --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ ++ --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ ++ --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ ++ --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ ++ --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ ++ --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ ++ --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ ++ --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ ++ --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ ++ --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ ++ --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ ++ --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ ++ --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ ++ --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ ++ --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ ++ --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ ++ --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ ++ --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ ++ --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ ++ --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ ++ --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ ++ --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ ++ --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ ++ --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ ++ --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ ++ --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ ++ --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ ++ --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ ++ --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ ++ --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ ++ --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ ++ --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ ++ --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ ++ --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ ++ --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ ++ --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ ++ --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ ++ --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd ++ # via ++ # aiohttp ++ # aiosignal ++google-api-core[grpc]==2.30.0 \ ++ --hash=sha256:02edfa9fab31e17fc0befb5f161b3bf93c9096d99aed584625f38065c511ad9b \ ++ --hash=sha256:80be49ee937ff9aba0fd79a6eddfde35fe658b9953ab9b79c57dd7061afa8df5 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++ # google-cloud-bigquery ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-core ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++ # google-cloud-storage ++ # google-cloud-videointelligence ++ # google-cloud-vision ++google-apitools==0.5.31 \ ++ --hash=sha256:4af0dd6dd4582810690251f0b57a97c1873dadfda54c5bc195844c8907624170 \ ++ --hash=sha256:6be92c1c3e93485450420bb0e365d47eb4d8a835d03ebe1963dc6da4d39a7b0e ++ # via apache-beam ++google-auth[requests]==2.49.0 \ ++ --hash=sha256:9cc2d9259d3700d7a257681f81052db6737495a1a46b610597f4b8bafe5286ae \ ++ --hash=sha256:f893ef7307f19cf53700b7e2f61b5a6affe3aa0edf9943b13788920ab92d8d87 ++ # via ++ # apache-beam ++ # cloud-sql-python-connector ++ # google-api-core ++ # google-auth-httplib2 ++ # google-cloud-aiplatform ++ # google-cloud-bigquery ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-core ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-storage ++ # google-cloud-videointelligence ++ # google-cloud-vision ++ # google-genai ++ # keyrings-google-artifactregistry-auth ++google-auth-httplib2==0.2.1 \ ++ --hash=sha256:1be94c611db91c01f9703e7f62b0a59bbd5587a95571c7b6fade510d648bc08b \ ++ --hash=sha256:5ef03be3927423c87fb69607b42df23a444e434ddb2555b73b3679793187b7de ++ # via apache-beam ++google-cloud-aiplatform==1.140.0 \ ++ --hash=sha256:e94493a2682b9d17efa7146a53bb3665bf1595c3394fd3d0f45d18f71623fddc \ ++ --hash=sha256:ea7eb1870b4cf600f8c2472102e21c3a1bcaf723d6e49f00ed51bc6b88d54fff ++ # via apache-beam ++google-cloud-bigquery==3.40.1 \ ++ --hash=sha256:75afcfb6e007238fe1deefb2182105249321145ff921784fe7b1de2b4ba24506 \ ++ --hash=sha256:9082a6b8193aba87bed6a2c79cf1152b524c99bb7e7ac33a785e333c09eac868 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++google-cloud-bigquery-storage==2.36.2 \ ++ --hash=sha256:823a73db0c4564e8ad3eedcfd5049f3d5aa41775267863b5627211ec36be2dbf \ ++ --hash=sha256:ad49d8c09ad6cd82da4efe596fcfcdbc1458bf05b93915e3c5c00f1e700ae128 ++ # via ++ # -r python/default_base_bqmonitor_requirements.txt ++ # apache-beam ++google-cloud-bigtable==2.35.0 \ ++ --hash=sha256:f355bfce1f239453ec2bb3839b0f4f9937cf34ef06ef29e1ca63d58fd38d0c50 \ ++ --hash=sha256:f5699012c5fea4bd4bdf7e80e5e3a812a847eb8f41bf8dc2f43095d6d876b83b ++ # via apache-beam ++google-cloud-core==2.5.0 \ ++ --hash=sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc \ ++ --hash=sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963 ++ # via ++ # apache-beam ++ # google-cloud-bigquery ++ # google-cloud-bigtable ++ # google-cloud-datastore ++ # google-cloud-spanner ++ # google-cloud-storage ++google-cloud-datastore==2.23.0 \ ++ --hash=sha256:24a1b1d29b902148fe41b109699f76fd3aa60591e9d547c0f8b87d7bf9ff213f \ ++ --hash=sha256:80049883a4ae928fdcc661ba6803ec267665dc0e6f3ce2da91441079a6bb6387 ++ # via apache-beam ++google-cloud-dlp==3.34.0 \ ++ --hash=sha256:3a1a7fd335fd65641ac3cb3f24f96ee9345d546d413ad6c88071a59404b1a641 \ ++ --hash=sha256:6dfa3172520d5a7fa8ccce47a9622cde815f037b4aa6fb6d69984fd597bf8007 ++ # via apache-beam ++google-cloud-kms==3.11.0 \ ++ --hash=sha256:07f2829e4ed986220802d013219fe159ecbdecec35907a6ddeea37ea9daecd8d \ ++ --hash=sha256:5f7d7bdb347f13a8a2b7bad6cbdf3846a51690df7215586845b62851b88839f7 ++ # via apache-beam ++google-cloud-language==2.19.0 \ ++ --hash=sha256:3b88f6eabd1c2413a1c6c918cbe40a22a5d14401930309717dbb709b353c6c64 \ ++ --hash=sha256:a43044632c8aada30a9c3246e00bfc867a56188be0c6e08e8764731296a05e0b ++ # via apache-beam ++google-cloud-monitoring==2.29.1 \ ++ --hash=sha256:86cac55cdd2608561819d19544fb3c129bbb7dcecc445d8de426e34cd6fa8e49 \ ++ --hash=sha256:944a57031f20da38617d184d5658c1f938e019e8061f27fd944584831a1b9d5a ++ # via google-cloud-spanner ++google-cloud-pubsub==2.35.0 \ ++ --hash=sha256:2c0d1d7ccda52fa12fb73f34b7eb9899381e2fd931c7d47b10f724cdfac06f95 \ ++ --hash=sha256:c32e4eb29e532ec784b5abb5d674807715ec07895b7c022b9404871dec09970d ++ # via ++ # apache-beam ++ # google-cloud-pubsublite ++google-cloud-pubsublite==1.13.0 \ ++ --hash=sha256:00773be42f335ec0e76e0e3e6c72041c2795268433f48add29780cea41e8bd3e \ ++ --hash=sha256:cc56ca57755e7665a66f0c0025ca923f7bfeb39ba408859ffe87cb840c0e82b5 ++ # via apache-beam ++google-cloud-recommendations-ai==0.10.18 \ ++ --hash=sha256:a6bccb45744fd89f038aa3e19502d1f46ea61c438dd2c08528533f8e185ec469 \ ++ --hash=sha256:c5c4b569d8be96e65dc273d18a35e44147ef62f845c8a9e8afd93474802c60c8 ++ # via apache-beam ++google-cloud-resource-manager==1.16.0 \ ++ --hash=sha256:cc938f87cc36c2672f062b1e541650629e0d954c405a4dac35ceedee70c267c3 \ ++ --hash=sha256:fb9a2ad2b5053c508e1c407ac31abfd1a22e91c32876c1892830724195819a28 ++ # via google-cloud-aiplatform ++google-cloud-secret-manager==2.26.0 \ ++ --hash=sha256:0d1d6f76327685a0ed78a4cf50f289e1bfbbe56026ed0affa98663b86d6d50d6 \ ++ --hash=sha256:940a5447a6ec9951446fd1a0f22c81a4303fde164cd747aae152c5f5c8e6723e ++ # via apache-beam ++google-cloud-spanner==3.63.0 \ ++ --hash=sha256:6ffae0ed589bbbd2d8831495e266198f3d069005cfe65c664448c9a727c88e7b \ ++ --hash=sha256:e2a4fb3bdbad4688645f455d498705d3f935b7c9011f5c94c137b77569b47a62 ++ # via apache-beam ++google-cloud-storage==2.19.0 \ ++ --hash=sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba \ ++ --hash=sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++google-cloud-videointelligence==2.18.0 \ ++ --hash=sha256:2cf4a32f64f4e01fdfb78b7bf625aa82df9129c87854796348887eac60290e95 \ ++ --hash=sha256:b2ae39bd22d186218684a297c2fa2fa636e5874e69d39f719504d729f44639fd ++ # via apache-beam ++google-cloud-vision==3.12.1 \ ++ --hash=sha256:8c661bc0e7a6bd3d03a1a645b977af24ae3f21ccf3df8e213298659fd0d40813 \ ++ --hash=sha256:f99b83af7588d30e708b87e09ff73e43e380497fe82c799b9f05e03f310027c8 ++ # via apache-beam ++google-crc32c==1.8.0 \ ++ --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ ++ --hash=sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a \ ++ --hash=sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff \ ++ --hash=sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288 \ ++ --hash=sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411 \ ++ --hash=sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a \ ++ --hash=sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15 \ ++ --hash=sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb \ ++ --hash=sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa \ ++ --hash=sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962 \ ++ --hash=sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215 \ ++ --hash=sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b \ ++ --hash=sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27 \ ++ --hash=sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113 \ ++ --hash=sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f \ ++ --hash=sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f \ ++ --hash=sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d \ ++ --hash=sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2 \ ++ --hash=sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092 \ ++ --hash=sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7 \ ++ --hash=sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2 \ ++ --hash=sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93 \ ++ --hash=sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8 \ ++ --hash=sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21 \ ++ --hash=sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79 \ ++ --hash=sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2 \ ++ --hash=sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc \ ++ --hash=sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454 \ ++ --hash=sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2 \ ++ --hash=sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733 \ ++ --hash=sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697 \ ++ --hash=sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651 \ ++ --hash=sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c ++ # via ++ # google-cloud-bigtable ++ # google-cloud-storage ++ # google-resumable-media ++google-genai==1.66.0 \ ++ --hash=sha256:7f127a39cf695277104ce4091bb26e417c59bb46e952ff3699c3a982d9c474ee \ ++ --hash=sha256:ffc01647b65046bca6387320057aa51db0ad64bcc72c8e3e914062acfa5f7c49 ++ # via google-cloud-aiplatform ++google-resumable-media==2.8.0 \ ++ --hash=sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582 \ ++ --hash=sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae ++ # via ++ # google-cloud-bigquery ++ # google-cloud-storage ++googleapis-common-protos[grpc]==1.73.0 \ ++ --hash=sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a \ ++ --hash=sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8 ++ # via ++ # google-api-core ++ # grpc-google-iam-v1 ++ # grpcio-status ++grpc-google-iam-v1==0.14.3 \ ++ --hash=sha256:7a7f697e017a067206a3dfef44e4c634a34d3dee135fe7d7a4613fe3e59217e6 \ ++ --hash=sha256:879ac4ef33136c5491a6300e27575a9ec760f6cdf9a2518798c1b8977a5dc389 ++ # via ++ # google-cloud-bigtable ++ # google-cloud-kms ++ # google-cloud-pubsub ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++grpc-interceptor==0.15.4 \ ++ --hash=sha256:0035f33228693ed3767ee49d937bac424318db173fef4d2d0170b3215f254d9d \ ++ --hash=sha256:1f45c0bcb58b6f332f37c637632247c9b02bc6af0fdceb7ba7ce8d2ebbfb0926 ++ # via google-cloud-spanner ++grpcio==1.65.5 \ ++ --hash=sha256:05f02d68fc720e085f061b704ee653b181e6d5abfe315daef085719728d3d1fd \ ++ --hash=sha256:078038e150a897e5e402ed3d57f1d31ebf604cbed80f595bd281b5da40762a92 \ ++ --hash=sha256:0b2944390a496567de9e70418f3742b477d85d8ca065afa90432edc91b4bb8ad \ ++ --hash=sha256:11f8b16121768c1cb99d7dcb84e01510e60e6a206bf9123e134118802486f035 \ ++ --hash=sha256:1c4caafe71aef4dabf53274bbf4affd6df651e9f80beedd6b8e08ff438ed3260 \ ++ --hash=sha256:1cbc208edb9acf1cc339396a1a36b83796939be52f34e591c90292045b579fbf \ ++ --hash=sha256:238a625f391a1b9f5f069bdc5930f4fd71b74426bea52196fc7b83f51fa97d34 \ ++ --hash=sha256:2a6d8169812932feac514b420daffae8ab8e36f90f3122b94ae767e633296b17 \ ++ --hash=sha256:2b91ce647b6307f25650872454a4d02a2801f26a475f90d0b91ed8110baae589 \ ++ --hash=sha256:3207ae60d07e5282c134b6e02f9271a2cb523c6d7a346c6315211fe2bf8d61ed \ ++ --hash=sha256:32d60e18ff7c34fe3f6db3d35ad5c6dc99f5b43ff3982cb26fad4174462d10b1 \ ++ --hash=sha256:33158e56c6378063923c417e9fbdb28660b6e0e2835af42e67f5a7793f587af7 \ ++ --hash=sha256:47d0aaaab82823f0aa6adea5184350b46e2252e13a42a942db84da5b733f2e05 \ ++ --hash=sha256:55714ea852396ec9568f45f487639945ab674de83c12bea19d5ddbc3ae41ada3 \ ++ --hash=sha256:6c4e62bcf297a1568f627f39576dbfc27f1e5338a691c6dd5dd6b3979da51d1c \ ++ --hash=sha256:76991b7a6fb98630a3328839755181ce7c1aa2b1842aa085fd4198f0e5198960 \ ++ --hash=sha256:770bd4bd721961f6dd8049bc27338564ba8739913f77c0f381a9815e465ff965 \ ++ --hash=sha256:7a412959aa5f08c5ac04aa7b7c3c041f5e4298cadd4fcc2acff195b56d185ebc \ ++ --hash=sha256:84c901cdec16a092099f251ef3360d15e29ef59772150fa261d94573612539b5 \ ++ --hash=sha256:85ae8f8517d5bcc21fb07dbf791e94ed84cc28f84c903cdc2bd7eaeb437c8f45 \ ++ --hash=sha256:89c00a18801b1ed9cc441e29b521c354725d4af38c127981f2c950c796a09b6e \ ++ --hash=sha256:8da58ff80bc4556cf29bc03f5fff1f03b8387d6aaa7b852af9eb65b2cf833be4 \ ++ --hash=sha256:8e5c4c15ac3fe1eb68e46bc51e66ad29be887479f231f8237cf8416058bf0cc1 \ ++ --hash=sha256:a101696f9ece90a0829988ff72f1b1ea2358f3df035bdf6d675dd8b60c2c0894 \ ++ --hash=sha256:a2f80510f99f82d4eb825849c486df703f50652cea21c189eacc2b84f2bde764 \ ++ --hash=sha256:a70a20eed87bba647a38bedd93b3ce7db64b3f0e8e0952315237f7f5ca97b02d \ ++ --hash=sha256:a80e9a5e3f93c54f5eb82a3825ea1fc4965b2fa0026db2abfecb139a5c4ecdf1 \ ++ --hash=sha256:ab5ec837d8cee8dbce9ef6386125f119b231e4333cc6b6d57b6c5c7c82a72331 \ ++ --hash=sha256:b67d450f1e008fedcd81e097a3a400a711d8be1a8b20f852a7b8a73fead50fe3 \ ++ --hash=sha256:b7ca419f1462390851eec395b2089aad1e49546b52d4e2c972ceb76da69b10f8 \ ++ --hash=sha256:b8270b15b99781461b244f5c81d5c2bc9696ab9189fb5ff86c841417fb3b39fe \ ++ --hash=sha256:bc74f3f745c37e2c5685c9d2a2d5a94de00f286963f5213f763ae137bf4f2358 \ ++ --hash=sha256:c3655139d7be213c32c79ef6fb2367cae28e56ef68e39b1961c43214b457f257 \ ++ --hash=sha256:c97962720489ef31b5ad8a916e22bc31bba3664e063fb9f6702dce056d4aa61b \ ++ --hash=sha256:cabd706183ee08d8026a015af5819a0b3a8959bdc9d1f6fdacd1810f09200f2a \ ++ --hash=sha256:d3a9e35bcb045e39d7cac30464c285389b9a816ac2067e4884ad2c02e709ef8e \ ++ --hash=sha256:d750e9330eb14236ca11b78d0c494eed13d6a95eb55472298f0e547c165ee324 \ ++ --hash=sha256:d7df567b67d16d4177835a68d3f767bbcbad04da9dfb52cbd19171f430c898bd \ ++ --hash=sha256:ec6f219fb5d677a522b0deaf43cea6697b16f338cb68d009e30930c4aa0d2209 \ ++ --hash=sha256:ec71fc5b39821ad7d80db7473c8f8c2910f3382f0ddadfbcfc2c6c437107eb67 \ ++ --hash=sha256:ee6ed64a27588a2c94e8fa84fe8f3b5c89427d4d69c37690903d428ec61ca7e4 \ ++ --hash=sha256:f17f9fa2d947dbfaca01b3ab2c62eefa8240131fdc67b924eb42ce6032e3e5c1 \ ++ --hash=sha256:f5b5970341359341d0e4c789da7568264b2a89cd976c05ea476036852b5950cd \ ++ --hash=sha256:f79c87c114bf37adf408026b9e2e333fe9ff31dfc9648f6f80776c513145c813 \ ++ --hash=sha256:fa36dd8496d3af0d40165252a669fa4f6fd2db4b4026b9a9411cbf060b9d6a15 \ ++ --hash=sha256:fe6505376f5b00bb008e4e1418152e3ad3d954b629da286c7913ff3cfc0ff740 ++ # via ++ # apache-beam ++ # google-api-core ++ # google-cloud-bigquery-storage ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-videointelligence ++ # google-cloud-vision ++ # googleapis-common-protos ++ # grpc-google-iam-v1 ++ # grpc-interceptor ++ # grpcio-status ++grpcio-status==1.65.5 \ ++ --hash=sha256:2c9fa3af32efd26f01837d44305dce106973bc5357b9a9fc8bbd87bb8bf833d1 \ ++ --hash=sha256:44a445ce55375545a913e005be36fbec7999a4cc320d7aecb7a4469d3d49366c ++ # via ++ # google-api-core ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++grpclib==0.4.9 \ ++ --hash=sha256:7762ec1c8ed94dfad597475152dd35cbd11aecaaca2f243e29702435ca24cf0e \ ++ --hash=sha256:cc589c330fa81004c6400a52a566407574498cb5b055fa927013361e21466c46 ++ # via betterproto ++h11==0.16.0 \ ++ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ ++ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 ++ # via httpcore ++h2==4.3.0 \ ++ --hash=sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1 \ ++ --hash=sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd ++ # via grpclib ++hpack==4.1.0 \ ++ --hash=sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496 \ ++ --hash=sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca ++ # via h2 ++httpcore==1.0.9 \ ++ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ ++ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 ++ # via httpx ++httplib2==0.22.0 \ ++ --hash=sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc \ ++ --hash=sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81 ++ # via ++ # apache-beam ++ # google-apitools ++ # google-auth-httplib2 ++ # oauth2client ++httpx==0.28.1 \ ++ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ ++ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad ++ # via google-genai ++hyperframe==6.1.0 \ ++ --hash=sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5 \ ++ --hash=sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08 ++ # via h2 ++idna==3.11 \ ++ --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ ++ --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 ++ # via ++ # anyio ++ # httpx ++ # requests ++ # yarl ++importlib-metadata==8.7.1 \ ++ --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \ ++ --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151 ++ # via ++ # keyring ++ # opentelemetry-api ++jaraco-classes==3.4.0 \ ++ --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ ++ --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 ++ # via keyring ++jaraco-context==6.1.1 \ ++ --hash=sha256:0df6a0287258f3e364072c3e40d5411b20cafa30cb28c4839d24319cecf9f808 \ ++ --hash=sha256:bc046b2dc94f1e5532bd02402684414575cc11f565d929b6563125deb0a6e581 ++ # via keyring ++jaraco-functools==4.4.0 \ ++ --hash=sha256:9eec1e36f45c818d9bf307c8948eb03b2b56cd44087b3cdc989abca1f20b9176 \ ++ --hash=sha256:da21933b0417b89515562656547a77b4931f98176eb173644c0d35032a33d6bb ++ # via keyring ++jeepney==0.9.0 \ ++ --hash=sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683 \ ++ --hash=sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732 ++ # via ++ # keyring ++ # secretstorage ++jsonpickle==3.4.2 \ ++ --hash=sha256:2efa2778859b6397d5804b0a98d52cd2a7d9a70fcb873bc5a3ca5acca8f499ba \ ++ --hash=sha256:fd6c273278a02b3b66e3405db3dd2f4dbc8f4a4a3123bfcab3045177c6feb9c3 ++ # via apache-beam ++keyring==25.7.0 \ ++ --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f \ ++ --hash=sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b ++ # via keyrings-google-artifactregistry-auth ++keyrings-google-artifactregistry-auth==1.1.2 \ ++ --hash=sha256:bd6abb72740d2dfeb4a5c03c3b105c6f7dba169caa29dee3959694f1f02c77de \ ++ --hash=sha256:e3f18b50fa945c786593014dc225810d191671d4f5f8e12d9259e39bad3605a3 ++ # via apache-beam ++mmh3==5.2.1 \ ++ --hash=sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d \ ++ --hash=sha256:0634581290e6714c068f4aa24020acf7880927d1f0084fa753d9799ae9610082 \ ++ --hash=sha256:08043f7cb1fb9467c3fbbbaea7896986e7fbc81f4d3fd9289a73d9110ab6207a \ ++ --hash=sha256:0a3984146e414684a6be2862d84fcb1035f4984851cb81b26d933bab6119bf00 \ ++ --hash=sha256:0bbc17250b10d3466875a40a52520a6bac3c02334ca709207648abd3c223ed5c \ ++ --hash=sha256:0cc21533878e5586b80d74c281d7f8da7932bc8ace50b8d5f6dbf7e3935f63f1 \ ++ --hash=sha256:0d0b7e803191db5f714d264044e06189c8ccd3219e936cc184f07106bd17fd7b \ ++ --hash=sha256:113f78e7463a36dbbcea05bfe688efd7fa759d0f0c56e73c974d60dcfec3dfcc \ ++ --hash=sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104 \ ++ --hash=sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8 \ ++ --hash=sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9 \ ++ --hash=sha256:1d9f9a3ce559a5267014b04b82956993270f63ec91765e13e9fd73daf2d2738e \ ++ --hash=sha256:1e4ecee40ba19e6975e1120829796770325841c2f153c0e9aecca927194c6a2a \ ++ --hash=sha256:22b0f9971ec4e07e8223f2beebe96a6cfc779d940b6f27d26604040dd74d3a44 \ ++ --hash=sha256:26fb5b9c3946bf7f1daed7b37e0c03898a6f062149127570f8ede346390a0825 \ ++ --hash=sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4 \ ++ --hash=sha256:28cfab66577000b9505a0d068c731aee7ca85cd26d4d63881fab17857e0fe1fb \ ++ --hash=sha256:29bc3973676ae334412efdd367fcd11d036b7be3efc1ce2407ef8676dabfeb82 \ ++ --hash=sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f \ ++ --hash=sha256:2d5d542bf2abd0fd0361e8017d03f7cb5786214ceb4a40eef1539d6585d93386 \ ++ --hash=sha256:30e4d2084df019880d55f6f7bea35328d9b464ebee090baa372c096dc77556fb \ ++ --hash=sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d \ ++ --hash=sha256:368625fb01666655985391dbad3860dc0ba7c0d6b9125819f3121ee7292b4ac8 \ ++ --hash=sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593 \ ++ --hash=sha256:3a9fed49c6ce4ed7e73f13182760c65c816da006debe67f37635580dfb0fae00 \ ++ --hash=sha256:3c38d142c706201db5b2345166eeef1e7740e3e2422b470b8ba5c8727a9b4c7a \ ++ --hash=sha256:3cb61db880ec11e984348227b333259994c2c85caa775eb7875decb3768db890 \ ++ --hash=sha256:3d74a03fb57757ece25aa4b3c1c60157a1cece37a020542785f942e2f827eed5 \ ++ --hash=sha256:3f796b535008708846044c43302719c6956f39ca2d93f2edda5319e79a29efbb \ ++ --hash=sha256:41105377f6282e8297f182e393a79cfffd521dde37ace52b106373bdcd9ca5cb \ ++ --hash=sha256:41aac7002a749f08727cb91babff1daf8deac317c0b1f317adc69be0e6c375d1 \ ++ --hash=sha256:44983e45310ee5b9f73397350251cdf6e63a466406a105f1d16cb5baa659270b \ ++ --hash=sha256:4cbbde66f1183db040daede83dd86c06d663c5bb2af6de1142b7c8c37923dd74 \ ++ --hash=sha256:4eda76074cfca2787c8cf1bec603eaebdddd8b061ad5502f85cddae998d54f00 \ ++ --hash=sha256:4fc6cd65dc4d2fdb2625e288939a3566e36127a84811a4913f02f3d5931da52d \ ++ --hash=sha256:50885073e2909251d4718634a191c49ae5f527e5e1736d738e365c3e8be8f22b \ ++ --hash=sha256:5174a697ce042fa77c407e05efe41e03aa56dae9ec67388055820fb48cf4c3ba \ ++ --hash=sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b \ ++ --hash=sha256:54fe8518abe06a4c3852754bfd498b30cc58e667f376c513eac89a244ce781a4 \ ++ --hash=sha256:55dbbd8ffbc40d1697d5e2d0375b08599dae8746b0b08dea05eee4ce81648fac \ ++ --hash=sha256:57b52603e89355ff318025dd55158f6e71396c0f1f609d548e9ea9c94cc6ce0a \ ++ --hash=sha256:58370d05d033ee97224c81263af123dea3d931025030fd34b61227a768a8858a \ ++ --hash=sha256:5d87a3584093e1a89987e3d36d82c98d9621b2cb944e22a420aa1401e096758f \ ++ --hash=sha256:623f938f6a039536cc02b7582a07a080f13fdfd48f87e63201d92d7e34d09a18 \ ++ --hash=sha256:62815d2c67f2dd1be76a253d88af4e1da19aeaa1820146dec52cf8bee2958b16 \ ++ --hash=sha256:6290289fa5fb4c70fd7f72016e03633d60388185483ff3b162912c81205ae2cf \ ++ --hash=sha256:67e41a497bac88cc1de96eeba56eeb933c39d54bc227352f8455aa87c4ca4000 \ ++ --hash=sha256:6c85c38a279ca9295a69b9b088a2e48aa49737bb1b34e6a9dc6297c110e8d912 \ ++ --hash=sha256:6f01f044112d43a20be2f13a11683666d87151542ad627fe41a18b9791d2802f \ ++ --hash=sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5 \ ++ --hash=sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e \ ++ --hash=sha256:72d1cc63bcc91e14933f77d51b3df899d6a07d184ec515ea7f56bff659e124d7 \ ++ --hash=sha256:7374d6e3ef72afe49697ecd683f3da12f4fc06af2d75433d0580c6746d2fa025 \ ++ --hash=sha256:7501e9be34cb21e72fcfe672aafd0eee65c16ba2afa9dcb5500a587d3a0580f0 \ ++ --hash=sha256:76219cd1eefb9bf4af7856e3ae563d15158efa145c0aab01e9933051a1954045 \ ++ --hash=sha256:7aec798c2b01aaa65a55f1124f3405804184373abb318a3091325aece235f67c \ ++ --hash=sha256:7be6dfb49e48fd0a7d91ff758a2b51336f1cd21f9d44b20f6801f072bd080cdd \ ++ --hash=sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d \ ++ --hash=sha256:7e8ec5f606e0809426d2440e0683509fb605a8820a21ebd120dcdba61b74ef7f \ ++ --hash=sha256:7f196cd7910d71e9d9860da0ff7a77f64d22c1ad931f1dd18559a06e03109fc0 \ ++ --hash=sha256:82f3802bfc4751f420d591c5c864de538b71cea117fce67e4595c2afede08a15 \ ++ --hash=sha256:85ffc9920ffc39c5eee1e3ac9100c913a0973996fbad5111f939bbda49204bb7 \ ++ --hash=sha256:8e6c219e375f6341d0959af814296372d265a8ca1af63825f65e2e87c618f006 \ ++ --hash=sha256:8f767ba0911602ddef289404e33835a61168314ebd3c729833db2ed685824211 \ ++ --hash=sha256:8ff038d52ef6aa0f309feeba00c5095c9118d0abf787e8e8454d6048db2037fc \ ++ --hash=sha256:915e7a2418f10bd1151b1953df06d896db9783c9cfdb9a8ee1f9b3a4331ab503 \ ++ --hash=sha256:92883836caf50d5255be03d988d75bc93e3f86ba247b7ca137347c323f731deb \ ++ --hash=sha256:960b1b3efa39872ac8b6cc3a556edd6fb90ed74f08c9c45e028f1005b26aa55d \ ++ --hash=sha256:9aeaf53eaa075dd63e81512522fd180097312fb2c9f476333309184285c49ce0 \ ++ --hash=sha256:9d8089d853c7963a8ce87fff93e2a67075c0bc08684a08ea6ad13577c38ffc38 \ ++ --hash=sha256:a4130d0b9ce5fad6af07421b1aecc7e079519f70d6c05729ab871794eded8617 \ ++ --hash=sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f \ ++ --hash=sha256:add7ac388d1e0bf57259afbcf9ed05621a3bf11ce5ee337e7536f1e1aaf056b0 \ ++ --hash=sha256:b1f12bd684887a0a5d55e6363ca87056f361e45451105012d329b86ec19dbe0b \ ++ --hash=sha256:b3f99e1756fc48ad507b95e5d86f2fb21b3d495012ff13e6592ebac14033f166 \ ++ --hash=sha256:b4cce60d0223074803c9dbe0721ad3fa51dafe7d462fee4b656a1aa01ee07518 \ ++ --hash=sha256:baeb47635cb33375dee4924cd93d7f5dcaa786c740b08423b0209b824a1ee728 \ ++ --hash=sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad \ ++ --hash=sha256:bbfcb95d9a744e6e2827dfc66ad10e1020e0cac255eb7f85652832d5a264c2fc \ ++ --hash=sha256:bd6e7d363aa93bd3421b30b6af97064daf47bc96005bddba67c5ffbc6df426b8 \ ++ --hash=sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03 \ ++ --hash=sha256:c302245fd6c33d96bd169c7ccf2513c20f4c1e417c07ce9dce107c8bc3f8411f \ ++ --hash=sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2 \ ++ --hash=sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229 \ ++ --hash=sha256:cd471ede0d802dd936b6fab28188302b2d497f68436025857ca72cd3810423fe \ ++ --hash=sha256:d106493a60dcb4aef35a0fac85105e150a11cf8bc2b0d388f5a33272d756c966 \ ++ --hash=sha256:d30b650595fdbe32366b94cb14f30bb2b625e512bd4e1df00611f99dc5c27fd4 \ ++ --hash=sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6 \ ++ --hash=sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1 \ ++ --hash=sha256:d771f085fcdf4035786adfb1d8db026df1eb4b41dac1c3d070d1e49512843227 \ ++ --hash=sha256:dae0f0bd7d30c0ad61b9a504e8e272cb8391eed3f1587edf933f4f6b33437450 \ ++ --hash=sha256:db0562c5f71d18596dcd45e854cf2eeba27d7543e1a3acdafb7eef728f7fe85d \ ++ --hash=sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b \ ++ --hash=sha256:e080c0637aea036f35507e803a4778f119a9b436617694ae1c5c366805f1e997 \ ++ --hash=sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6 \ ++ --hash=sha256:e8b4b5580280b9265af3e0409974fb79c64cf7523632d03fbf11df18f8b0181e \ ++ --hash=sha256:e8b5378de2b139c3a830f0209c1e91f7705919a4b3e563a10955104f5097a70a \ ++ --hash=sha256:e904f2417f0d6f6d514f3f8b836416c360f306ddaee1f84de8eef1e722d212e5 \ ++ --hash=sha256:eee884572b06bbe8a2b54f424dbd996139442cf83c76478e1ec162512e0dd2c7 \ ++ --hash=sha256:f1fbb0a99125b1287c6d9747f937dc66621426836d1a2d50d05aecfc81911b57 \ ++ --hash=sha256:f40a95186a72fa0b67d15fef0f157bfcda00b4f59c8a07cbe5530d41ac35d105 \ ++ --hash=sha256:f6e0bfe77d238308839699944164b96a2eeccaf55f2af400f54dc20669d8d5f2 \ ++ --hash=sha256:f963eafc0a77a6c0562397da004f5876a9bcf7265a7bcc3205e29636bc4a1312 \ ++ --hash=sha256:fb9d44c25244e11c8be3f12c938ca8ba8404620ef8092245d2093c6ab3df260f \ ++ --hash=sha256:fc78739b5ec6e4fb02301984a3d442a91406e7700efbe305071e7fd1c78278f2 \ ++ --hash=sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a \ ++ --hash=sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b ++ # via google-cloud-spanner ++more-itertools==10.8.0 \ ++ --hash=sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b \ ++ --hash=sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd ++ # via ++ # jaraco-classes ++ # jaraco-functools ++multidict==6.7.1 \ ++ --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ ++ --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ ++ --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \ ++ --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \ ++ --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \ ++ --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \ ++ --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \ ++ --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \ ++ --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \ ++ --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \ ++ --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \ ++ --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \ ++ --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \ ++ --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \ ++ --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \ ++ --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \ ++ --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \ ++ --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \ ++ --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \ ++ --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \ ++ --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \ ++ --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \ ++ --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \ ++ --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \ ++ --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \ ++ --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \ ++ --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \ ++ --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \ ++ --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \ ++ --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \ ++ --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \ ++ --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \ ++ --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \ ++ --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \ ++ --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \ ++ --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \ ++ --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \ ++ --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \ ++ --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \ ++ --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \ ++ --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \ ++ --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \ ++ --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \ ++ --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \ ++ --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \ ++ --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \ ++ --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \ ++ --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \ ++ --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \ ++ --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \ ++ --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \ ++ --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \ ++ --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \ ++ --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \ ++ --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \ ++ --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \ ++ --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \ ++ --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \ ++ --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \ ++ --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \ ++ --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \ ++ --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \ ++ --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \ ++ --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \ ++ --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \ ++ --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \ ++ --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \ ++ --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \ ++ --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \ ++ --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \ ++ --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \ ++ --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \ ++ --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \ ++ --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \ ++ --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \ ++ --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \ ++ --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \ ++ --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \ ++ --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \ ++ --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \ ++ --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \ ++ --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \ ++ --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \ ++ --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \ ++ --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \ ++ --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \ ++ --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \ ++ --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \ ++ --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \ ++ --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \ ++ --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \ ++ --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \ ++ --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \ ++ --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \ ++ --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \ ++ --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \ ++ --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \ ++ --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \ ++ --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \ ++ --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \ ++ --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \ ++ --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \ ++ --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \ ++ --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \ ++ --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \ ++ --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \ ++ --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \ ++ --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \ ++ --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \ ++ --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \ ++ --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \ ++ --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \ ++ --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \ ++ --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \ ++ --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \ ++ --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \ ++ --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \ ++ --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \ ++ --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \ ++ --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \ ++ --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \ ++ --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \ ++ --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \ ++ --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \ ++ --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \ ++ --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \ ++ --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \ ++ --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \ ++ --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \ ++ --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \ ++ --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \ ++ --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \ ++ --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \ ++ --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \ ++ --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \ ++ --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \ ++ --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \ ++ --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \ ++ --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \ ++ --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \ ++ --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \ ++ --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \ ++ --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \ ++ --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \ ++ --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \ ++ --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19 ++ # via ++ # aiohttp ++ # grpclib ++ # yarl ++numpy==2.4.2 \ ++ --hash=sha256:00ab83c56211a1d7c07c25e3217ea6695e50a3e2f255053686b081dc0b091a82 \ ++ --hash=sha256:068cdb2d0d644cdb45670810894f6a0600797a69c05f1ac478e8d31670b8ee75 \ ++ --hash=sha256:0f01dcf33e73d80bd8dc0f20a71303abbafa26a19e23f6b68d1aa9990af90257 \ ++ --hash=sha256:0fece1d1f0a89c16b03442eae5c56dc0be0c7883b5d388e0c03f53019a4bfd71 \ ++ --hash=sha256:12e26134a0331d8dbd9351620f037ec470b7c75929cb8a1537f6bfe411152a1a \ ++ --hash=sha256:1ae241bbfc6ae276f94a170b14785e561cb5e7f626b6688cf076af4110887413 \ ++ --hash=sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181 \ ++ --hash=sha256:209fae046e62d0ce6435fcfe3b1a10537e858249b3d9b05829e2a05218296a85 \ ++ --hash=sha256:20abd069b9cda45874498b245c8015b18ace6de8546bf50dfa8cea1696ed06ef \ ++ --hash=sha256:21982668592194c609de53ba4933a7471880ccbaadcc52352694a59ecc860b3a \ ++ --hash=sha256:25f2059807faea4b077a2b6837391b5d830864b3543627f381821c646f31a63c \ ++ --hash=sha256:2653de5c24910e49c2b106499803124dde62a5a1fe0eedeaecf4309a5f639390 \ ++ --hash=sha256:2b8f157c8a6f20eb657e240f8985cc135598b2b46985c5bccbde7616dc9c6b1e \ ++ --hash=sha256:2fb882da679409066b4603579619341c6d6898fc83a8995199d5249f986e8e8f \ ++ --hash=sha256:40397bda92382fcec844066efb11f13e1c9a3e2a8e8f318fb72ed8b6db9f60f1 \ ++ --hash=sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b \ ++ --hash=sha256:47c5a6ed21d9452b10227e5e8a0e1c22979811cad7dcc19d8e3e2fb8fa03f1a3 \ ++ --hash=sha256:4f069069931240b3fc703f1e23df63443dbd6390614c8c44a87d96cd0ec81eb1 \ ++ --hash=sha256:52b913ec40ff7ae845687b0b34d8d93b60cb66dcee06996dd5c99f2fc9328657 \ ++ --hash=sha256:5633c0da313330fd20c484c78cdd3f9b175b55e1a766c4a174230c6b70ad8262 \ ++ --hash=sha256:5daf6f3914a733336dab21a05cdec343144600e964d2fcdabaac0c0269874b2a \ ++ --hash=sha256:5eea80d908b2c1f91486eb95b3fb6fab187e569ec9752ab7d9333d2e66bf2d6b \ ++ --hash=sha256:602f65afdef699cda27ec0b9224ae5dc43e328f4c24c689deaf77133dbee74d0 \ ++ --hash=sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae \ ++ --hash=sha256:66cb9422236317f9d44b67b4d18f44efe6e9c7f8794ac0462978513359461554 \ ++ --hash=sha256:6d82351358ffbcdcd7b686b90742a9b86632d6c1c051016484fa0b326a0a1548 \ ++ --hash=sha256:6e9f61981ace1360e42737e2bae58b27bf28a1b27e781721047d84bd754d32e7 \ ++ --hash=sha256:6ed0be1ee58eef41231a5c943d7d1375f093142702d5723ca2eb07db9b934b05 \ ++ --hash=sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1 \ ++ --hash=sha256:7df2de1e4fba69a51c06c28f5a3de36731eb9639feb8e1cf7e4a7b0daf4cf622 \ ++ --hash=sha256:7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1 \ ++ --hash=sha256:7f54844851cdb630ceb623dcec4db3240d1ac13d4990532446761baede94996a \ ++ --hash=sha256:805cc8de9fd6e7a22da5aed858e0ab16be5a4db6c873dde1d7451c541553aa27 \ ++ --hash=sha256:8906e71fd8afcb76580404e2a950caef2685df3d2a57fe82a86ac8d33cc007ba \ ++ --hash=sha256:89f7268c009bc492f506abd6f5265defa7cb3f7487dc21d357c3d290add45082 \ ++ --hash=sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443 \ ++ --hash=sha256:8e4549f8a3c6d13d55041925e912bfd834285ef1dd64d6bc7d542583355e2e98 \ ++ --hash=sha256:8e9afaeb0beff068b4d9cd20d322ba0ee1cecfb0b08db145e4ab4dd44a6b5110 \ ++ --hash=sha256:98f16a80e917003a12c0580f97b5f875853ebc33e2eaa4bccfc8201ac6869308 \ ++ --hash=sha256:9e35d3e0144137d9fdae62912e869136164534d64a169f86438bc9561b6ad49f \ ++ --hash=sha256:9e4424677ce4b47fe73c8b5556d876571f7c6945d264201180db2dc34f676ab5 \ ++ --hash=sha256:adb6ed2ad29b9e15321d167d152ee909ec73395901b70936f029c3bc6d7f4460 \ ++ --hash=sha256:aea4f66ff44dfddf8c2cffd66ba6538c5ec67d389285292fe428cb2c738c8aef \ ++ --hash=sha256:b21041e8cb6a1eb5312dd1d2f80a94d91efffb7a06b70597d44f1bd2dfc315ab \ ++ --hash=sha256:b2f0073ed0868db1dcd86e052d37279eef185b9c8db5bf61f30f46adac63c909 \ ++ --hash=sha256:b3a24467af63c67829bfaa61eecf18d5432d4f11992688537be59ecd6ad32f5e \ ++ --hash=sha256:b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695 \ ++ --hash=sha256:bba37bc29d4d85761deed3954a1bc62be7cf462b9510b51d367b769a8c8df325 \ ++ --hash=sha256:bd3a7a9f5847d2fb8c2c6d1c862fa109c31a9abeca1a3c2bd5a64572955b2979 \ ++ --hash=sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0 \ ++ --hash=sha256:c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32 \ ++ --hash=sha256:c3cd545784805de05aafe1dde61752ea49a359ccba9760c1e5d1c88a93bbf2b7 \ ++ --hash=sha256:c7ac672d699bf36275c035e16b65539931347d68b70667d28984c9fb34e07fa7 \ ++ --hash=sha256:cb7bbb88aa74908950d979eeaa24dbdf1a865e3c7e45ff0121d8f70387b55f73 \ ++ --hash=sha256:cd2bd2bbed13e213d6b55dc1d035a4f91748a7d3edc9480c13898b0353708920 \ ++ --hash=sha256:cda077c2e5b780200b6b3e09d0b42205a3d1c68f30c6dceb90401c13bff8fe74 \ ++ --hash=sha256:cf28c0c1d4c4bf00f509fa7eb02c58d7caf221b50b467bcb0d9bbf1584d5c821 \ ++ --hash=sha256:d0d9b7c93578baafcbc5f0b83eaf17b79d345c6f36917ba0c67f45226911d499 \ ++ --hash=sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000 \ ++ --hash=sha256:d30291931c915b2ab5717c2974bb95ee891a1cf22ebc16a8006bd59cd210d40a \ ++ --hash=sha256:d9f64d786b3b1dd742c946c42d15b07497ed14af1a1f3ce840cce27daa0ce913 \ ++ --hash=sha256:da6cad4e82cb893db4b69105c604d805e0c3ce11501a55b5e9f9083b47d2ffe8 \ ++ --hash=sha256:df1b10187212b198dd45fa943d8985a3c8cf854aed4923796e0e019e113a1bda \ ++ --hash=sha256:e04ae107ac591763a47398bb45b568fc38f02dbc4aa44c063f67a131f99346cb \ ++ --hash=sha256:e6dee3bb76aa4009d5a912180bf5b2de012532998d094acee25d9cb8dee3e44a \ ++ --hash=sha256:e7e88598032542bd49af7c4747541422884219056c268823ef6e5e89851c8825 \ ++ --hash=sha256:e98c97502435b53741540a5717a6749ac2ada901056c7db951d33e11c885cc7d \ ++ --hash=sha256:ec055f6dae239a6299cace477b479cca2fc125c5675482daf1dd886933a1076f \ ++ --hash=sha256:f74f0f7779cc7ae07d1810aab8ac6b1464c3eafb9e283a40da7309d5e6e48fbb \ ++ --hash=sha256:fbde1b0c6e81d56f5dccd95dd4a711d9b95df1ae4009a60887e56b27e8d903fa \ ++ --hash=sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236 \ ++ --hash=sha256:fd49860271d52127d61197bb50b64f58454e9f578cb4b2c001a6de8b1f50b0b1 ++ # via apache-beam ++oauth2client==4.1.3 \ ++ --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ ++ --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 ++ # via google-apitools ++objsize==0.7.1 \ ++ --hash=sha256:634a0c134c4b1ff2c340fe29caf58bc0a16cb2ff7c556df609d04f026fdf4eca \ ++ --hash=sha256:91e68d2a3031efb61b0e8cb7f995ddaeb65fe5ace9e737785e029f0932c2e619 ++ # via apache-beam ++opentelemetry-api==1.40.0 \ ++ --hash=sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f \ ++ --hash=sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9 ++ # via ++ # google-cloud-pubsub ++ # google-cloud-spanner ++ # opentelemetry-resourcedetector-gcp ++ # opentelemetry-sdk ++ # opentelemetry-semantic-conventions ++opentelemetry-resourcedetector-gcp==1.11.0a0 \ ++ --hash=sha256:5d65a2a039b1d40c6f41421dbb08d5f441368275ac6de6e76a8fccd1f6acb67e \ ++ --hash=sha256:915a1d6fd15daca9eedd3fc52b0f705375054f2ef140e2e7a6b4cca95a47cdb1 ++ # via google-cloud-spanner ++opentelemetry-sdk==1.40.0 \ ++ --hash=sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2 \ ++ --hash=sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1 ++ # via ++ # google-cloud-pubsub ++ # google-cloud-spanner ++ # opentelemetry-resourcedetector-gcp ++opentelemetry-semantic-conventions==0.61b0 \ ++ --hash=sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a \ ++ --hash=sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2 ++ # via ++ # google-cloud-spanner ++ # opentelemetry-sdk ++orjson==3.11.7 \ ++ --hash=sha256:043d3006b7d32c7e233b8cfb1f01c651013ea079e08dcef7189a29abd8befe11 \ ++ --hash=sha256:0527a4510c300e3b406591b0ba69b5dc50031895b0a93743526a3fc45f59d26e \ ++ --hash=sha256:0724e265bc548af1dedebd9cb3d24b4e1c1e685a343be43e87ba922a5c5fff2f \ ++ --hash=sha256:136dcd6a2e796dfd9ffca9fc027d778567b0b7c9968d092842d3c323cef88aa8 \ ++ --hash=sha256:14f440c7268c8f8633d1b3d443a434bd70cb15686117ea6beff8fdc8f5917a1e \ ++ --hash=sha256:1d98b30cc1313d52d4af17d9c3d307b08389752ec5f2e5febdfada70b0f8c733 \ ++ --hash=sha256:1eb80451a9c351a71dfaf5b7ccc13ad065405217726b59fdbeadbcc544f9d223 \ ++ --hash=sha256:1ee5cc7160a821dfe14f130bc8e63e7611051f964b463d9e2a3a573204446a4d \ ++ --hash=sha256:23d6c20517a97a9daf1d48b580fcdc6f0516c6f4b5038823426033690b4d2650 \ ++ --hash=sha256:26c3b9132f783b7d7903bf1efb095fed8d4a3a85ec0d334ee8beff3d7a4749d5 \ ++ --hash=sha256:31c80ce534ac4ea3739c5ee751270646cbc46e45aea7576a38ffec040b4029a1 \ ++ --hash=sha256:3726be79e36e526e3d9c1aceaadbfb4a04ee80a72ab47b3f3c17fefb9812e7b8 \ ++ --hash=sha256:390a1dce0c055ddf8adb6aa94a73b45a4a7d7177b5c584b8d1c1947f2ba60fb3 \ ++ --hash=sha256:3a2479753bbb95b0ebcf7969f562cdb9668e6d12416a35b0dda79febf89cdea2 \ ++ --hash=sha256:3c4bc6c6ac52cdaa267552544c73e486fecbd710b7ac09bc024d5a78555a22f6 \ ++ --hash=sha256:411ebaf34d735e25e358a6d9e7978954a9c9d58cfb47bc6683cdc3964cd2f910 \ ++ --hash=sha256:4682d1db3bcebd2b64757e0ddf9e87ae5f00d29d16c5cdf3a62f561d08cc3dd2 \ ++ --hash=sha256:4a2e9c5be347b937a2e0203866f12bba36082e89b402ddb9e927d5822e43088d \ ++ --hash=sha256:57036b27ac8a25d81112eb0cc9835cd4833c5b16e1467816adc0015f59e870dc \ ++ --hash=sha256:5ede977b5fe5ac91b1dffc0a517ca4542d2ec8a6a4ff7b2652d94f640796342a \ ++ --hash=sha256:5fdfad2093bdd08245f2e204d977facd5f871c88c4a71230d5bcbd0e43bf6222 \ ++ --hash=sha256:623ad1b9548ef63886319c16fa317848e465a21513b31a6ad7b57443c3e0dcf5 \ ++ --hash=sha256:652c6c3af76716f4a9c290371ba2e390ede06f6603edb277b481daf37f6f464e \ ++ --hash=sha256:6543001328aa857187f905308a028935864aefe9968af3848401b6fe80dbb471 \ ++ --hash=sha256:6e776b998ac37c0396093d10290e60283f59cfe0fc3fccbd0ccc4bd04dd19892 \ ++ --hash=sha256:71924496986275a737f38e3f22b4e0878882b3f7a310d2ff4dc96e812789120c \ ++ --hash=sha256:733ae23ada68b804b222c44affed76b39e30806d38660bf1eb200520d259cc16 \ ++ --hash=sha256:7477aa6a6ec6139c5cb1cc7b214643592169a5494d200397c7fc95d740d5fcf3 \ ++ --hash=sha256:79cacb0b52f6004caf92405a7e1f11e6e2de8bdf9019e4f76b44ba045125cd6b \ ++ --hash=sha256:7ba61079379b0ae29e117db13bda5f28d939766e410d321ec1624afc6a0b0504 \ ++ --hash=sha256:800988273a014a0541483dc81021247d7eacb0c845a9d1a34a422bc718f41539 \ ++ --hash=sha256:814be4b49b228cfc0b3c565acf642dd7d13538f966e3ccde61f4f55be3e20785 \ ++ --hash=sha256:845c3e0d8ded9c9271cd79596b9b552448b885b97110f628fb687aee2eed11c1 \ ++ --hash=sha256:849e38203e5be40b776ed2718e587faf204d184fc9a008ae441f9442320c0cab \ ++ --hash=sha256:89e13dd3f89f1c38a9c9eba5fbf7cdc2d1feca82f5f290864b4b7a6aac704576 \ ++ --hash=sha256:89e440ebc74ce8ab5c7bc4ce6757b4a6b1041becb127df818f6997b5c71aa60b \ ++ --hash=sha256:8ff206156006da5b847c9304b6308a01e8cdbc8cce824e2779a5ba71c3def141 \ ++ --hash=sha256:91c81ef070c8f3220054115e1ef468b1c9ce8497b4e526cb9f68ab4dc0a7ac62 \ ++ --hash=sha256:9487abc2c2086e7c8eb9a211d2ce8855bae0e92586279d0d27b341d5ad76c85c \ ++ --hash=sha256:962d046ee1765f74a1da723f4b33e3b228fe3a48bd307acce5021dfefe0e29b2 \ ++ --hash=sha256:996b65230271f1a97026fd0e6a753f51fbc0c335d2ad0c6201f711b0da32693b \ ++ --hash=sha256:9b1a67243945819ce55d24a30b59d6a168e86220452d2c96f4d1f093e71c0c49 \ ++ --hash=sha256:9c0b51672e466fd7e56230ffbae7f1639e18d0ce023351fb75da21b71bc2c960 \ ++ --hash=sha256:9e54f3808e2b6b945078c41aa8d9b5834b28c50843846e97807e5adb75fa9705 \ ++ --hash=sha256:a02c833f38f36546ba65a452127633afce4cf0dd7296b753d3bb54e55e5c0174 \ ++ --hash=sha256:a12b80df61aab7b98b490fe9e4879925ba666fccdfcd175252ce4d9035865ace \ ++ --hash=sha256:a16bcd08ab0bcdfc7e8801d9c4a9cc17e58418e4d48ddc6ded4e9e4b1a94062b \ ++ --hash=sha256:a56df3239294ea5964adf074c54bcc4f0ccd21636049a2cf3ca9cf03b5d03cf1 \ ++ --hash=sha256:a709e881723c9b18acddcfb8ba357322491ad553e277cf467e1e7e20e2d90561 \ ++ --hash=sha256:ab49d4b2a6a1d415ddb9f37a21e02e0d5dbfe10b7870b21bf779fc21e9156157 \ ++ --hash=sha256:ae9e0b37a834cef7ce8f99de6498f8fad4a2c0bf6bfc3d02abd8ed56aa15b2de \ ++ --hash=sha256:b4a9eefdc70bf8bf9857f0290f973dec534ac84c35cd6a7f4083be43e7170a8f \ ++ --hash=sha256:b63c6e6738d7c3470ad01601e23376aa511e50e1f3931395b9f9c722406d1a67 \ ++ --hash=sha256:b7b1dae39230a393df353827c855a5f176271c23434cfd2db74e0e424e693e10 \ ++ --hash=sha256:b8d14b71c0b12963fe8a62aac87119f1afdf4cb88a400f61ca5ae581449efcb5 \ ++ --hash=sha256:b9f95dcdea9d4f805daa9ddf02617a89e484c6985fa03055459f90e87d7a0757 \ ++ --hash=sha256:b9fc4d0f81f394689e0814617aadc4f2ea0e8025f38c226cbf22d3b5ddbf025d \ ++ --hash=sha256:bd03ea7606833655048dab1a00734a2875e3e86c276e1d772b2a02556f0d895f \ ++ --hash=sha256:bd0d68edd7dfca1b2eca9361a44ac9f24b078de3481003159929a0573f21a6bf \ ++ --hash=sha256:bda117c4148e81f746655d5a3239ae9bd00cb7bc3ca178b5fc5a5997e9744183 \ ++ --hash=sha256:bf742e149121dc5648ba0a08ea0871e87b660467ef168a3a5e53bc1fbd64bb74 \ ++ --hash=sha256:c2428d358d85e8da9d37cba18b8c4047c55222007a84f97156a5b22028dfbfc0 \ ++ --hash=sha256:c2e85fe4698b6a56d5e2ebf7ae87544d668eb6bde1ad1226c13f44663f20ec9e \ ++ --hash=sha256:c43b8b5bab288b6b90dac410cca7e986a4fa747a2e8f94615aea407da706980d \ ++ --hash=sha256:cededd6738e1c153530793998e31c05086582b08315db48ab66649768f326baa \ ++ --hash=sha256:d06e5c5fed5caedd2e540d62e5b1c25e8c82431b9e577c33537e5fa4aa909539 \ ++ --hash=sha256:d772afdb22555f0c58cfc741bdae44180122b3616faa1ecadb595cd526e4c993 \ ++ --hash=sha256:d897e81f8d0cbd2abb82226d1860ad2e1ab3ff16d7b08c96ca00df9d45409ef4 \ ++ --hash=sha256:de0a37f21d0d364954ad5de1970491d7fbd0fb1ef7417d4d56a36dc01ba0c0a0 \ ++ --hash=sha256:e7745312efa9e11c17fbd3cb3097262d079da26930ae9ae7ba28fb738367cbad \ ++ --hash=sha256:ed46f17096e28fb28d2975834836a639af7278aa87c84f68ab08fbe5b8bd75fa \ ++ --hash=sha256:f4f7c956b5215d949a1f65334cf9d7612dde38f20a95f2315deef167def91a6f \ ++ --hash=sha256:f50979824bde13d32b4320eedd513431c921102796d86be3eee0b58e58a3ecd1 \ ++ --hash=sha256:f904c24bdeabd4298f7a977ef14ca2a022ca921ed670b92ecd16ab6f3d01f867 ++ # via apache-beam ++overrides==7.7.0 \ ++ --hash=sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a \ ++ --hash=sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49 ++ # via google-cloud-pubsublite ++packaging==26.0 \ ++ --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ ++ --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 ++ # via ++ # apache-beam ++ # google-cloud-aiplatform ++ # google-cloud-bigquery ++pg8000==1.31.5 \ ++ --hash=sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201 \ ++ --hash=sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78 ++ # via apache-beam ++pluggy==1.6.0 \ ++ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ ++ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 ++ # via keyrings-google-artifactregistry-auth ++propcache==0.4.1 \ ++ --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e \ ++ --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 \ ++ --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be \ ++ --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 \ ++ --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 \ ++ --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b \ ++ --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 \ ++ --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf \ ++ --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 \ ++ --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 \ ++ --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 \ ++ --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 \ ++ --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 \ ++ --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 \ ++ --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 \ ++ --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 \ ++ --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 \ ++ --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc \ ++ --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe \ ++ --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb \ ++ --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 \ ++ --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 \ ++ --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e \ ++ --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff \ ++ --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 \ ++ --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 \ ++ --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 \ ++ --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 \ ++ --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf \ ++ --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 \ ++ --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a \ ++ --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc \ ++ --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a \ ++ --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 \ ++ --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 \ ++ --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 \ ++ --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 \ ++ --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 \ ++ --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 \ ++ --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af \ ++ --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa \ ++ --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 \ ++ --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc \ ++ --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 \ ++ --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 \ ++ --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf \ ++ --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 \ ++ --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 \ ++ --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c \ ++ --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 \ ++ --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e \ ++ --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 \ ++ --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 \ ++ --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 \ ++ --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 \ ++ --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db \ ++ --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f \ ++ --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d \ ++ --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 \ ++ --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e \ ++ --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 \ ++ --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 \ ++ --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a \ ++ --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b \ ++ --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f \ ++ --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f \ ++ --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 \ ++ --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 \ ++ --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 \ ++ --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 \ ++ --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db \ ++ --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded \ ++ --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb \ ++ --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 \ ++ --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 \ ++ --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 \ ++ --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 \ ++ --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 \ ++ --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f \ ++ --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b \ ++ --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 \ ++ --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 \ ++ --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 \ ++ --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c \ ++ --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 \ ++ --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 \ ++ --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 \ ++ --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 \ ++ --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 \ ++ --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 \ ++ --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db \ ++ --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac \ ++ --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 \ ++ --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 \ ++ --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f \ ++ --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 \ ++ --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 \ ++ --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 \ ++ --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 \ ++ --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a \ ++ --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f \ ++ --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 \ ++ --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 \ ++ --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 \ ++ --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 \ ++ --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 \ ++ --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 \ ++ --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 \ ++ --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 \ ++ --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 \ ++ --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e \ ++ --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 \ ++ --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d \ ++ --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 \ ++ --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c \ ++ --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e \ ++ --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 \ ++ --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af \ ++ --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f \ ++ --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 \ ++ --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 \ ++ --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 ++ # via ++ # aiohttp ++ # yarl ++proto-plus==1.27.1 \ ++ --hash=sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147 \ ++ --hash=sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc ++ # via ++ # apache-beam ++ # google-api-core ++ # google-cloud-aiplatform ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-pubsublite ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++ # google-cloud-videointelligence ++ # google-cloud-vision ++protobuf==5.29.6 \ ++ --hash=sha256:36ade6ff88212e91aef4e687a971a11d7d24d6948a66751abc1b3238648f5d05 \ ++ --hash=sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1 \ ++ --hash=sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86 \ ++ --hash=sha256:76e07e6567f8baf827137e8d5b8204b6c7b6488bbbff1bf0a72b383f77999c18 \ ++ --hash=sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda \ ++ --hash=sha256:831e2da16b6cc9d8f1654c041dd594eda43391affd3c03a91bea7f7f6da106d6 \ ++ --hash=sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6 \ ++ --hash=sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269 \ ++ --hash=sha256:cb4c86de9cd8a7f3a256b9744220d87b847371c6b2f10bde87768918ef33ba49 \ ++ --hash=sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723 \ ++ --hash=sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9 ++ # via ++ # apache-beam ++ # google-api-core ++ # google-cloud-aiplatform ++ # google-cloud-bigquery-storage ++ # google-cloud-bigtable ++ # google-cloud-datastore ++ # google-cloud-dlp ++ # google-cloud-kms ++ # google-cloud-language ++ # google-cloud-monitoring ++ # google-cloud-pubsub ++ # google-cloud-recommendations-ai ++ # google-cloud-resource-manager ++ # google-cloud-secret-manager ++ # google-cloud-spanner ++ # google-cloud-videointelligence ++ # google-cloud-vision ++ # googleapis-common-protos ++ # grpc-google-iam-v1 ++ # grpcio-status ++ # proto-plus ++pyarrow==18.1.0 \ ++ --hash=sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe \ ++ --hash=sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e \ ++ --hash=sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54 \ ++ --hash=sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99 \ ++ --hash=sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e \ ++ --hash=sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9 \ ++ --hash=sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181 \ ++ --hash=sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76 \ ++ --hash=sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c \ ++ --hash=sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c \ ++ --hash=sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56 \ ++ --hash=sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754 \ ++ --hash=sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b \ ++ --hash=sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9 \ ++ --hash=sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992 \ ++ --hash=sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc \ ++ --hash=sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7 \ ++ --hash=sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa \ ++ --hash=sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b \ ++ --hash=sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73 \ ++ --hash=sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812 \ ++ --hash=sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d \ ++ --hash=sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052 \ ++ --hash=sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191 \ ++ --hash=sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386 \ ++ --hash=sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324 \ ++ --hash=sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4 \ ++ --hash=sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba \ ++ --hash=sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470 \ ++ --hash=sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71 \ ++ --hash=sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30 \ ++ --hash=sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33 \ ++ --hash=sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a \ ++ --hash=sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8 \ ++ --hash=sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee \ ++ --hash=sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c \ ++ --hash=sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6 \ ++ --hash=sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854 \ ++ --hash=sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0 \ ++ --hash=sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21 \ ++ --hash=sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2 \ ++ --hash=sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c ++ # via apache-beam ++pyarrow-hotfix==0.7 \ ++ --hash=sha256:3236f3b5f1260f0e2ac070a55c1a7b339c4bb7267839bd2015e283234e758100 \ ++ --hash=sha256:59399cd58bdd978b2e42816a4183a55c6472d4e33d183351b6069f11ed42661d ++ # via apache-beam ++pyasn1==0.6.2 \ ++ --hash=sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf \ ++ --hash=sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b ++ # via ++ # oauth2client ++ # pyasn1-modules ++ # rsa ++pyasn1-modules==0.4.2 \ ++ --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ ++ --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 ++ # via ++ # google-auth ++ # oauth2client ++pycparser==3.0 \ ++ --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ ++ --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 ++ # via cffi ++pydantic==2.12.5 \ ++ --hash=sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49 \ ++ --hash=sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d ++ # via ++ # google-cloud-aiplatform ++ # google-genai ++pydantic-core==2.41.5 \ ++ --hash=sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90 \ ++ --hash=sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740 \ ++ --hash=sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504 \ ++ --hash=sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84 \ ++ --hash=sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33 \ ++ --hash=sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c \ ++ --hash=sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0 \ ++ --hash=sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e \ ++ --hash=sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0 \ ++ --hash=sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a \ ++ --hash=sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34 \ ++ --hash=sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2 \ ++ --hash=sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3 \ ++ --hash=sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815 \ ++ --hash=sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14 \ ++ --hash=sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba \ ++ --hash=sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375 \ ++ --hash=sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf \ ++ --hash=sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963 \ ++ --hash=sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1 \ ++ --hash=sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808 \ ++ --hash=sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553 \ ++ --hash=sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1 \ ++ --hash=sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2 \ ++ --hash=sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5 \ ++ --hash=sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470 \ ++ --hash=sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2 \ ++ --hash=sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b \ ++ --hash=sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660 \ ++ --hash=sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c \ ++ --hash=sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093 \ ++ --hash=sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5 \ ++ --hash=sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594 \ ++ --hash=sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008 \ ++ --hash=sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a \ ++ --hash=sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a \ ++ --hash=sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd \ ++ --hash=sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284 \ ++ --hash=sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586 \ ++ --hash=sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869 \ ++ --hash=sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294 \ ++ --hash=sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f \ ++ --hash=sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66 \ ++ --hash=sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51 \ ++ --hash=sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc \ ++ --hash=sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97 \ ++ --hash=sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a \ ++ --hash=sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d \ ++ --hash=sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9 \ ++ --hash=sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c \ ++ --hash=sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07 \ ++ --hash=sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36 \ ++ --hash=sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e \ ++ --hash=sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05 \ ++ --hash=sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e \ ++ --hash=sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941 \ ++ --hash=sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3 \ ++ --hash=sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612 \ ++ --hash=sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3 \ ++ --hash=sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b \ ++ --hash=sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe \ ++ --hash=sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146 \ ++ --hash=sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11 \ ++ --hash=sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60 \ ++ --hash=sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd \ ++ --hash=sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b \ ++ --hash=sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c \ ++ --hash=sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a \ ++ --hash=sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460 \ ++ --hash=sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1 \ ++ --hash=sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf \ ++ --hash=sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf \ ++ --hash=sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858 \ ++ --hash=sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2 \ ++ --hash=sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9 \ ++ --hash=sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2 \ ++ --hash=sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3 \ ++ --hash=sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6 \ ++ --hash=sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770 \ ++ --hash=sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d \ ++ --hash=sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc \ ++ --hash=sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23 \ ++ --hash=sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26 \ ++ --hash=sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa \ ++ --hash=sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8 \ ++ --hash=sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d \ ++ --hash=sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3 \ ++ --hash=sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d \ ++ --hash=sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034 \ ++ --hash=sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9 \ ++ --hash=sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1 \ ++ --hash=sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56 \ ++ --hash=sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b \ ++ --hash=sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c \ ++ --hash=sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a \ ++ --hash=sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e \ ++ --hash=sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9 \ ++ --hash=sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5 \ ++ --hash=sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a \ ++ --hash=sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556 \ ++ --hash=sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e \ ++ --hash=sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49 \ ++ --hash=sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2 \ ++ --hash=sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9 \ ++ --hash=sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b \ ++ --hash=sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc \ ++ --hash=sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb \ ++ --hash=sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0 \ ++ --hash=sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8 \ ++ --hash=sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82 \ ++ --hash=sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69 \ ++ --hash=sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b \ ++ --hash=sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c \ ++ --hash=sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75 \ ++ --hash=sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5 \ ++ --hash=sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f \ ++ --hash=sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad \ ++ --hash=sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b \ ++ --hash=sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7 \ ++ --hash=sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425 \ ++ --hash=sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52 ++ # via pydantic ++pymongo==4.16.0 \ ++ --hash=sha256:03f42396c1b2c6f46f5401c5b185adc25f6113716e16d9503977ee5386fca0fb \ ++ --hash=sha256:12762e7cc0f8374a8cae3b9f9ed8dabb5d438c7b33329232dd9b7de783454033 \ ++ --hash=sha256:15bb062c0d6d4b0be650410032152de656a2a9a2aa4e1a7443a22695afacb103 \ ++ --hash=sha256:19a1c96e7f39c7a59a9cfd4d17920cf9382f6f684faeff4649bf587dc59f8edc \ ++ --hash=sha256:1c01e8a7cd0ea66baf64a118005535ab5bf9f9eb63a1b50ac3935dccf9a54abe \ ++ --hash=sha256:1d638b0b1b294d95d0fdc73688a3b61e05cc4188872818cd240d51460ccabcb5 \ ++ --hash=sha256:21d02cc10a158daa20cb040985e280e7e439832fc6b7857bff3d53ef6914ad50 \ ++ --hash=sha256:2290909275c9b8f637b0a92eb9b89281e18a72922749ebb903403ab6cc7da914 \ ++ --hash=sha256:25a6b03a68f9907ea6ec8bc7cf4c58a1b51a18e23394f962a6402f8e46d41211 \ ++ --hash=sha256:2a3ba6be3d8acf64b77cdcd4e36f0e4a8e87965f14a8b09b90ca86f10a1dd2f2 \ ++ --hash=sha256:2b0714d7764efb29bf9d3c51c964aed7c4c7237b341f9346f15ceaf8321fdb35 \ ++ --hash=sha256:2cd60cd1e05de7f01927f8e25ca26b3ea2c09de8723241e5d3bcfdc70eaff76b \ ++ --hash=sha256:2d0082631a7510318befc2b4fdab140481eb4b9dd62d9245e042157085da2a70 \ ++ --hash=sha256:311d4549d6bf1f8c61d025965aebb5ba29d1481dc6471693ab91610aaffbc0eb \ ++ --hash=sha256:36ef2fee50eee669587d742fb456e349634b4fcf8926208766078b089054b24b \ ++ --hash=sha256:3ead8a0050c53eaa55935895d6919d393d0328ec24b2b9115bdbe881aa222673 \ ++ --hash=sha256:46ffb728d92dd5b09fc034ed91acf5595657c7ca17d4cf3751322cd554153c17 \ ++ --hash=sha256:4a19ea46a0fe71248965305a020bc076a163311aefbaa1d83e47d06fa30ac747 \ ++ --hash=sha256:4a9390dce61d705a88218f0d7b54d7e1fa1b421da8129fc7c009e029a9a6b81e \ ++ --hash=sha256:4c4872299ebe315a79f7f922051061634a64fda95b6b17677ba57ef00b2ba2a4 \ ++ --hash=sha256:4cd047ba6cc83cc24193b9208c93e134a985ead556183077678c59af7aacc725 \ ++ --hash=sha256:4d4f7ba040f72a9f43a44059872af5a8c8c660aa5d7f90d5344f2ed1c3c02721 \ ++ --hash=sha256:4d79aa147ce86aef03079096d83239580006ffb684eead593917186aee407767 \ ++ --hash=sha256:4fbb8d3552c2ad99d9e236003c0b5f96d5f05e29386ba7abae73949bfebc13dd \ ++ --hash=sha256:55f8d5a6fe2fa0b823674db2293f92d74cd5f970bc0360f409a1fc21003862d3 \ ++ --hash=sha256:5b9c6d689bbe5beb156374508133218610e14f8c81e35bc17d7a14e30ab593e6 \ ++ --hash=sha256:5d9fdb386cf958e6ef6ff537d6149be7edb76c3268cd6833e6c36aa447e4443f \ ++ --hash=sha256:60307bb91e0ab44e560fe3a211087748b2b5f3e31f403baf41f5b7b0a70bd104 \ ++ --hash=sha256:61567f712bda04c7545a037e3284b4367cad8d29b3dec84b4bf3b2147020a75b \ ++ --hash=sha256:66af44ed23686dd5422307619a6db4b56733c5e36fe8c4adf91326dcf993a043 \ ++ --hash=sha256:6af1aaa26f0835175d2200e62205b78e7ec3ffa430682e322cc91aaa1a0dbf28 \ ++ --hash=sha256:6b2a20edb5452ac8daa395890eeb076c570790dfce6b7a44d788af74c2f8cf96 \ ++ --hash=sha256:6f2077ec24e2f1248f9cac7b9a2dfb894e50cc7939fcebfb1759f99304caabef \ ++ --hash=sha256:77cfd37a43a53b02b7bd930457c7994c924ad8bbe8dff91817904bcbf291b371 \ ++ --hash=sha256:78037d02389745e247fe5ab0bcad5d1ab30726eaac3ad79219c7d6bbb07eec53 \ ++ --hash=sha256:7902882ed0efb7f0e991458ab3b8cf0eb052957264949ece2f09b63c58b04f78 \ ++ --hash=sha256:85dc2f3444c346ea019a371e321ac868a4fab513b7a55fe368f0cc78de8177cc \ ++ --hash=sha256:8a0f73af1ea56c422b2dcfc0437459148a799ef4231c6aee189d2d4c59d6728f \ ++ --hash=sha256:8a254d49a9ffe9d7f888e3c677eed3729b14ce85abb08cd74732cead6ccc3c66 \ ++ --hash=sha256:8ba8405065f6e258a6f872fe62d797a28f383a12178c7153c01ed04e845c600c \ ++ --hash=sha256:91899dd7fb9a8c50f09c3c1cf0cb73bfbe2737f511f641f19b9650deb61c00ca \ ++ --hash=sha256:91ac0cb0fe2bf17616c2039dac88d7c9a5088f5cb5829b27c9d250e053664d31 \ ++ --hash=sha256:92a232af9927710de08a6c16a9710cc1b175fb9179c0d946cd4e213b92b2a69a \ ++ --hash=sha256:948152b30eddeae8355495f9943a3bf66b708295c0b9b6f467de1c620f215487 \ ++ --hash=sha256:96aa7ab896889bf330209d26459e493d00f8855772a9453bfb4520bb1f495baf \ ++ --hash=sha256:9caacac0dd105e2555521002e2d17afc08665187017b466b5753e84c016628e6 \ ++ --hash=sha256:9d9885aad05f82fd7ea0c9ca505d60939746b39263fa273d0125170da8f59098 \ ++ --hash=sha256:9dc2c00bed568732b89e211b6adca389053d5e6d2d5a8979e80b813c3ec4d1f9 \ ++ --hash=sha256:a1bf44e13cf2d44d2ea2e928a8140d5d667304abe1a61c4d55b4906f389fbe64 \ ++ --hash=sha256:aa30cd16ddd2f216d07ba01d9635c873e97ddb041c61cf0847254edc37d1c60e \ ++ --hash=sha256:acda193f440dd88c2023cb00aa8bd7b93a9df59978306d14d87a8b12fe426b05 \ ++ --hash=sha256:bd4911c40a43a821dfd93038ac824b756b6e703e26e951718522d29f6eb166a8 \ ++ --hash=sha256:be1099a8295b1a722d03fb7b48be895d30f4301419a583dcf50e9045968a041c \ ++ --hash=sha256:c126fb72be2518395cc0465d4bae03125119136462e1945aea19840e45d89cfc \ ++ --hash=sha256:c53338613043038005bf2e41a2fafa08d29cdbc0ce80891b5366c819456c1ae9 \ ++ --hash=sha256:c789236366525c3ee3cd6e4e450a9ff629a7d1f4d88b8e18a0aea0615fd7ecf8 \ ++ --hash=sha256:cf0ec79e8ca7077f455d14d915d629385153b6a11abc0b93283ed73a8013e376 \ ++ --hash=sha256:d15f060bc6d0964a8bb70aba8f0cb6d11ae99715438f640cff11bbcf172eb0e8 \ ++ --hash=sha256:d284bf68daffc57516535f752e290609b3b643f4bd54b28fc13cb16a89a8bda6 \ ++ --hash=sha256:dabbf3c14de75a20cc3c30bf0c6527157224a93dfb605838eabb1a2ee3be008d \ ++ --hash=sha256:dbbc5b254c36c37d10abb50e899bc3939bbb7ab1e7c659614409af99bd3e7675 \ ++ --hash=sha256:dfc320f08ea9a7ec5b2403dc4e8150636f0d6150f4b9792faaae539c88e7db3b \ ++ --hash=sha256:e2d509786344aa844ae243f68f833ca1ac92ac3e35a92ae038e2ceb44aa355ef \ ++ --hash=sha256:e37469602473f41221cea93fd3736708f561f0fa08ab6b2873dd962014390d52 \ ++ --hash=sha256:ed162b2227f98d5b270ecbe1d53be56c8c81db08a1a8f5f02d89c7bb4d19591d \ ++ --hash=sha256:efe020c46ce3c3a89af6baec6569635812129df6fb6cf76d4943af3ba6ee2069 \ ++ --hash=sha256:f1c5f1f818b669875d191323a48912d3fcd2e4906410e8297bb09ac50c4d5ccc \ ++ --hash=sha256:f25001a955073b80510c0c3db0e043dbbc36904fd69e511c74e3d8640b8a5111 \ ++ --hash=sha256:f3867dc225d9423c245a51eaac2cfcd53dde8e0a8d8090bb6aed6e31bd6c2d4f \ ++ --hash=sha256:f513b2c6c0d5c491f478422f6b5b5c27ac1af06a54c93ef8631806f7231bd92e \ ++ --hash=sha256:f6e42c1bc985d9beee884780ae6048790eb4cd565c46251932906bdb1630034a ++ # via apache-beam ++pymysql==1.1.2 \ ++ --hash=sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03 \ ++ --hash=sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9 ++ # via apache-beam ++pyparsing==3.3.2 \ ++ --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ ++ --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc ++ # via httplib2 ++python-dateutil==2.9.0.post0 \ ++ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ ++ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 ++ # via ++ # apache-beam ++ # betterproto ++ # google-cloud-bigquery ++ # pg8000 ++python-tds==1.17.1 \ ++ --hash=sha256:35cb210b1a54e5ccc91570a83d4e9a2a16682cbeb00bede06fd6cdf9afa9762f \ ++ --hash=sha256:c97483a9adf1dcab8bee66e83429acc502753f389d134553edd818348b94ced0 ++ # via apache-beam ++pytz==2026.1.post1 \ ++ --hash=sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1 \ ++ --hash=sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a ++ # via apache-beam ++pyyaml==6.0.3 \ ++ --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ ++ --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ ++ --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ ++ --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ ++ --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ ++ --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ ++ --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ ++ --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ ++ --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ ++ --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ ++ --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ ++ --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ ++ --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ ++ --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ ++ --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ ++ --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ ++ --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ ++ --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ ++ --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ ++ --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ ++ --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ ++ --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ ++ --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ ++ --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ ++ --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ ++ --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ ++ --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ ++ --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ ++ --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ ++ --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ ++ --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ ++ --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ ++ --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ ++ --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ ++ --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ ++ --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ ++ --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ ++ --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ ++ --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ ++ --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ ++ --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ ++ --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ ++ --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ ++ --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ ++ --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ ++ --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ ++ --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ ++ --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ ++ --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ ++ --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ ++ --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ ++ --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ ++ --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ ++ --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ ++ --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ ++ --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ ++ --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ ++ --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ ++ --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ ++ --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ ++ --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ ++ --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ ++ --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ ++ --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ ++ --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ ++ --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ ++ --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ ++ --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ ++ --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ ++ --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ ++ --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ ++ --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ ++ --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 ++ # via apache-beam ++regex==2026.2.28 \ ++ --hash=sha256:00945d007fd74a9084d2ab79b695b595c6b7ba3698972fadd43e23230c6979c1 \ ++ --hash=sha256:00f2b8d9615aa165fdff0a13f1a92049bfad555ee91e20d246a51aa0b556c60a \ ++ --hash=sha256:01d65fd24206c8e1e97e2e31b286c59009636c022eb5d003f52760b0f42155d4 \ ++ --hash=sha256:02473c954af35dd2defeb07e44182f5705b30ea3f351a7cbffa9177beb14da5d \ ++ --hash=sha256:03a83cc26aa2acda6b8b9dfe748cf9e84cbd390c424a1de34fdcef58961a297a \ ++ --hash=sha256:09500be324f49b470d907b3ef8af9afe857f5cca486f853853f7945ddbf75911 \ ++ --hash=sha256:0b1d2b07614d95fa2bf8a63fd1e98bd8fa2b4848dc91b1efbc8ba219fdd73952 \ ++ --hash=sha256:0d25a10811de831c2baa6aef3c0be91622f44dd8d31dd12e69f6398efb15e48b \ ++ --hash=sha256:0d5bef2031cbf38757a0b0bc4298bb4824b6332d28edc16b39247228fbdbad97 \ ++ --hash=sha256:10d28e19bd4888e4abf43bd3925f3c134c52fdf7259219003588a42e24c2aa25 \ ++ --hash=sha256:180e08a435a0319e6a4821c3468da18dc7001987e1c17ae1335488dfe7518dd8 \ ++ --hash=sha256:195237dc327858a7721bf8b0bbbef797554bc13563c3591e91cd0767bacbe359 \ ++ --hash=sha256:19a9c9e0a8f24f39d575a6a854d516b48ffe4cbdcb9de55cb0570a032556ecff \ ++ --hash=sha256:1c2c95e1a2b0f89d01e821ff4de1be4b5d73d1f4b0bf679fa27c1ad8d2327f1a \ ++ --hash=sha256:1d367257cd86c1cbb97ea94e77b373a0bbc2224976e247f173d19e8f18b4afa7 \ ++ --hash=sha256:1e496956106fd59ba6322a8ea17141a27c5040e5ee8f9433ae92d4e5204462a0 \ ++ --hash=sha256:1f8b17be5c27a684ea6759983c13506bd77bfc7c0347dff41b18ce5ddd2ee09a \ ++ --hash=sha256:2234059cfe33d9813a3677ef7667999caea9eeaa83fef98eb6ce15c6cf9e0215 \ ++ --hash=sha256:25b6eb660c5cf4b8c3407a1ed462abba26a926cc9965e164268a3267bcc06a43 \ ++ --hash=sha256:2954379dd20752e82d22accf3ff465311cbb2bac6c1f92c4afd400e1757f7451 \ ++ --hash=sha256:2afa673660928d0b63d84353c6c08a8a476ddfc4a47e11742949d182e6863ce8 \ ++ --hash=sha256:2b2b23587b26496ff5fd40df4278becdf386813ec00dc3533fa43a4cf0e2ad3c \ ++ --hash=sha256:2fb950ac1d88e6b6a9414381f403797b236f9fa17e1eee07683af72b1634207b \ ++ --hash=sha256:3935174fa4d9f70525a4367aaff3cb8bc0548129d114260c29d9dfa4a5b41692 \ ++ --hash=sha256:39bb5727650b9a0275c6a6690f9bb3fe693a7e6cc5c3155b1240aedf8926423e \ ++ --hash=sha256:3b24bd7e9d85dc7c6a8bd2aa14ecd234274a0248335a02adeb25448aecdd420d \ ++ --hash=sha256:4390c365fd2d45278f45afd4673cb90f7285f5701607e3ad4274df08e36140ae \ ++ --hash=sha256:481df4623fa4969c8b11f3433ed7d5e3dc9cec0f008356c3212b3933fb77e3d8 \ ++ --hash=sha256:4f5c0b182ad4269e7381b7c27fdb0408399881f7a92a4624fd5487f2971dfc11 \ ++ --hash=sha256:50c2fc924749543e0eacc93ada6aeeb3ea5f6715825624baa0dccaec771668ae \ ++ --hash=sha256:511f7419f7afab475fd4d639d4aedfc54205bcb0800066753ef68a59f0f330b5 \ ++ --hash=sha256:516604edd17b1c2c3e579cf4e9b25a53bf8fa6e7cedddf1127804d3e0140ca64 \ ++ --hash=sha256:52b017b35ac2214d0db5f4f90e303634dc44e4aba4bd6235a27f97ecbe5b0472 \ ++ --hash=sha256:5a932ea8ad5d0430351ff9c76c8db34db0d9f53c1d78f06022a21f4e290c5c18 \ ++ --hash=sha256:5cdcc17d935c8f9d3f4db5c2ebe2640c332e3822ad5d23c2f8e0228e6947943a \ ++ --hash=sha256:5d10303dd18cedfd4d095543998404df656088240bcfd3cd20a8f95b861f74bd \ ++ --hash=sha256:5e68192bb3a1d6fb2836da24aa494e413ea65853a21505e142e5b1064a595f3d \ ++ --hash=sha256:64e7c6ad614573e0640f271e811a408d79a9e1fe62a46adb602f598df42a818d \ ++ --hash=sha256:6591f281cb44dc13de9585b552cec6fc6cf47fb2fe7a48892295ee9bc4a612f9 \ ++ --hash=sha256:69fc560ccbf08a09dc9b52ab69cacfae51e0ed80dc5693078bdc97db2f91ae96 \ ++ --hash=sha256:6d63a07e5ec8ce7184452cb00c41c37b49e67dc4f73b2955b5b8e782ea970784 \ ++ --hash=sha256:6db7bfae0f8a2793ff1f7021468ea55e2699d0790eb58ee6ab36ae43aa00bc5b \ ++ --hash=sha256:71a911098be38c859ceb3f9a9ce43f4ed9f4c6720ad8684a066ea246b76ad9ff \ ++ --hash=sha256:73cdcdbba8028167ea81490c7f45280113e41db2c7afb65a276f4711fa3bcbff \ ++ --hash=sha256:78454178c7df31372ea737996fb7f36b3c2c92cccc641d251e072478afb4babc \ ++ --hash=sha256:7900157786428a79615a8264dac1f12c9b02957c473c8110c6b1f972dcecaddf \ ++ --hash=sha256:7ab218076eb0944549e7fe74cf0e2b83a82edb27e81cc87411f76240865e04d5 \ ++ --hash=sha256:7c1b34dfa72f826f535b20712afa9bb3ba580020e834f3c69866c5bddbf10098 \ ++ --hash=sha256:851fa70df44325e1e4cdb79c5e676e91a78147b1b543db2aec8734d2add30ec2 \ ++ --hash=sha256:864cdd1a2ef5716b0ab468af40139e62ede1b3a53386b375ec0786bb6783fc05 \ ++ --hash=sha256:8710d61737b0c0ce6836b1da7109f20d495e49b3809f30e27e9560be67a257bf \ ++ --hash=sha256:9036b400b20e4858d56d117108d7813ed07bb7803e3eed766675862131135ca6 \ ++ --hash=sha256:9185cc63359862a6e80fe97f696e04b0ad9a11c4ac0a4a927f979f611bfe3768 \ ++ --hash=sha256:948c12ef30ecedb128903c2c2678b339746eb7c689c5c21957c4a23950c96d15 \ ++ --hash=sha256:94d63db12e45a9b9f064bfe4800cefefc7e5f182052e4c1b774d46a40ab1d9bb \ ++ --hash=sha256:96f6269a2882fbb0ee76967116b83679dc628e68eaea44e90884b8d53d833881 \ ++ --hash=sha256:97054c55db06ab020342cc0d35d6f62a465fa7662871190175f1ad6c655c028f \ ++ --hash=sha256:98adf340100cbe6fbaf8e6dc75e28f2c191b1be50ffefe292fb0e6f6eefdb0d8 \ ++ --hash=sha256:99985a2c277dcb9ccb63f937451af5d65177af1efdeb8173ac55b61095a0a05c \ ++ --hash=sha256:9b65d33a17101569f86d9c5966a8b1d7fbf8afdda5a8aa219301b0a80f58cf7d \ ++ --hash=sha256:9dd450db6458387167e033cfa80887a34c99c81d26da1bf8b0b41bf8c9cac88e \ ++ --hash=sha256:a25c7701e4f7a70021db9aaf4a4a0a67033c6318752146e03d1b94d32006217e \ ++ --hash=sha256:a448af01e3d8031c89c5d902040b124a5e921a25c4e5e07a861ca591ce429341 \ ++ --hash=sha256:a5dac14d0872eeb35260a8e30bac07ddf22adc1e3a0635b52b02e180d17c9c7e \ ++ --hash=sha256:a729e47d418ea11d03469f321aaf67cdee8954cde3ff2cf8403ab87951ad10f2 \ ++ --hash=sha256:aaffaecffcd2479ce87aa1e74076c221700b7c804e48e98e62500ee748f0f550 \ ++ --hash=sha256:b059e71ec363968671693a78c5053bd9cb2fe410f9b8e4657e88377ebd603a2e \ ++ --hash=sha256:b387a0d092dac157fb026d737dde35ff3e49ef27f285343e7c6401851239df27 \ ++ --hash=sha256:b389c61aa28a79c2e0527ac36da579869c2e235a5b208a12c5b5318cda2501d8 \ ++ --hash=sha256:b42f7466e32bf15a961cf09f35fa6323cc72e64d3d2c990b10de1274a5da0a59 \ ++ --hash=sha256:b49eb78048c6354f49e91e4b77da21257fecb92256b6d599ae44403cab30b05b \ ++ --hash=sha256:b5acd4b6a95f37c3c3828e5d053a7d4edaedb85de551db0153754924cb7c83e3 \ ++ --hash=sha256:b8b3f1be1738feadc69f62daa250c933e85c6f34fa378f54a7ff43807c1b9117 \ ++ --hash=sha256:b8cf76f1a29f0e99dcfd7aef1551a9827588aae5a737fe31442021165f1920dc \ ++ --hash=sha256:ba55c50f408fb5c346a3a02d2ce0ebc839784e24f7c9684fde328ff063c3cdea \ ++ --hash=sha256:bba2b18d70eeb7b79950f12f633beeecd923f7c9ad6f6bae28e59b4cb3ab046b \ ++ --hash=sha256:bbb882061f742eb5d46f2f1bd5304055be0a66b783576de3d7eef1bed4778a6e \ ++ --hash=sha256:bcb399ed84eabf4282587ba151f2732ad8168e66f1d3f85b1d038868fe547703 \ ++ --hash=sha256:bd477d5f79920338107f04aa645f094032d9e3030cc55be581df3d1ef61aa318 \ ++ --hash=sha256:bec23c11cbbf09a4df32fe50d57cbdd777bc442269b6e39a1775654f1c95dee2 \ ++ --hash=sha256:c0b5ccbb8ffb433939d248707d4a8b31993cb76ab1a0187ca886bf50e96df952 \ ++ --hash=sha256:c15af43c72a7fb0c97cbc66fa36a43546eddc5c06a662b64a0cbf30d6ac40944 \ ++ --hash=sha256:c7815afb0ca45456613fdaf60ea9c993715511c8d53a83bc468305cbc0ee23c7 \ ++ --hash=sha256:cb3b1db8ff6c7b8bf838ab05583ea15230cb2f678e569ab0e3a24d1e8320940b \ ++ --hash=sha256:d0b02e8b7e5874b48ae0f077ecca61c1a6a9f9895e9c6dfb191b55b242862033 \ ++ --hash=sha256:d6b08a06976ff4fb0d83077022fde3eca06c55432bb997d8c0495b9a4e9872f4 \ ++ --hash=sha256:d6cfe798d8da41bb1862ed6e0cba14003d387c3c0c4a5d45591076ae9f0ce2f8 \ ++ --hash=sha256:d8511a01d0e4ee1992eb3ba19e09bc1866fe03f05129c3aec3fdc4cbc77aad3f \ ++ --hash=sha256:dc8ed8c3f41c27acb83f7b6a9eb727a73fc6663441890c5cb3426a5f6a91ce7d \ ++ --hash=sha256:dd8847c4978bc3c7e6c826fb745f5570e518b8459ac2892151ce6627c7bc00d5 \ ++ --hash=sha256:de0cf053139f96219ccfabb4a8dd2d217c8c82cb206c91d9f109f3f552d6b43d \ ++ --hash=sha256:dee50f1be42222f89767b64b283283ef963189da0dda4a515aa54a5563c62dec \ ++ --hash=sha256:e1e7b24cb3ae9953a560c563045d1ba56ee4749fbd05cf21ba571069bd7be81b \ ++ --hash=sha256:e59bc8f30414d283ae8ee1617b13d8112e7135cb92830f0ec3688cb29152585a \ ++ --hash=sha256:e61eea47230eba62a31f3e8a0e3164d0f37ef9f40529fb2c79361bc6b53d2a92 \ ++ --hash=sha256:e621fb7c8dc147419b28e1702f58a0177ff8308a76fa295c71f3e7827849f5d9 \ ++ --hash=sha256:e71dcecaa113eebcc96622c17692672c2d104b1d71ddf7adeda90da7ddeb26fc \ ++ --hash=sha256:e7ce83654d1ab701cb619285a18a8e5a889c1216d746ddc710c914ca5fd71022 \ ++ --hash=sha256:e8c8cb2deba42f5ec1ede46374e990f8adc5e6456a57ac1a261b19be6f28e4e6 \ ++ --hash=sha256:ec0c608b7a7465ffadb344ed7c987ff2f11ee03f6a130b569aa74d8a70e8333c \ ++ --hash=sha256:ec6f5674c5dc836994f50f1186dd1fafde4be0666aae201ae2fcc3d29d8adf27 \ ++ --hash=sha256:edb1b1b3a5576c56f08ac46f108c40333f222ebfd5cf63afdfa3aab0791ebe5b \ ++ --hash=sha256:ef77bdde9c9eba3f7fa5b58084b29bbcc74bcf55fdbeaa67c102a35b5bd7e7cc \ ++ --hash=sha256:f2791948f7c70bb9335a9102df45e93d428f4b8128020d85920223925d73b9e1 \ ++ --hash=sha256:f467cb602f03fbd1ab1908f68b53c649ce393fde056628dc8c7e634dab6bfc07 \ ++ --hash=sha256:f8ed9a5d4612df9d4de15878f0bc6aa7a268afbe5af21a3fdd97fa19516e978c \ ++ --hash=sha256:fa539be029844c0ce1114762d2952ab6cfdd7c7c9bd72e0db26b94c3c36dcc5a \ ++ --hash=sha256:fb1c4ff62277d87a7335f2c1ea4e0387b8f2b3ad88a64efd9943906aafad4f33 \ ++ --hash=sha256:fb4db2f17e6484904f986c5a657cec85574c76b5c5e61c7aae9ffa1bc6224f95 \ ++ --hash=sha256:fb66e5245db9652abd7196ace599b04d9c0e4aa7c8f0e2803938377835780081 \ ++ --hash=sha256:fc48c500838be6882b32748f60a15229d2dea96e59ef341eaa96ec83538f498d \ ++ --hash=sha256:fcf26c3c6d0da98fada8ae4ef0aa1c3405a431c0a77eb17306d38a89b02adcd7 \ ++ --hash=sha256:fd0ce43e71d825b7c0661f9c54d4d74bd97c56c3fd102a8985bcfea48236bacb \ ++ --hash=sha256:fd63453f10d29097cc3dc62d070746523973fb5aa1c66d25f8558bebd47fed61 ++ # via apache-beam ++requests==2.32.5 \ ++ --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ ++ --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf ++ # via ++ # apache-beam ++ # cloud-sql-python-connector ++ # google-api-core ++ # google-auth ++ # google-cloud-bigquery ++ # google-cloud-storage ++ # google-genai ++ # keyrings-google-artifactregistry-auth ++ # opentelemetry-resourcedetector-gcp ++rsa==4.9.1 \ ++ --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ ++ --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 ++ # via ++ # google-auth ++ # oauth2client ++scramp==1.4.8 \ ++ --hash=sha256:87c2f15976845a2872fe5490a06097f0d01813cceb53774ea168c911f2ad025c \ ++ --hash=sha256:bd018fabfe46343cceeb9f1c3e8d23f55770271e777e3accbfaee3ff0a316e71 ++ # via pg8000 ++secretstorage==3.5.0 \ ++ --hash=sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137 \ ++ --hash=sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be ++ # via keyring ++six==1.17.0 \ ++ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ ++ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 ++ # via ++ # google-apitools ++ # oauth2client ++ # python-dateutil ++sniffio==1.3.1 \ ++ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ ++ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc ++ # via google-genai ++sortedcontainers==2.4.0 \ ++ --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \ ++ --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 ++ # via apache-beam ++sqlparse==0.5.5 \ ++ --hash=sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba \ ++ --hash=sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e ++ # via google-cloud-spanner ++tenacity==9.1.4 \ ++ --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 \ ++ --hash=sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a ++ # via google-genai ++typing-extensions==4.15.0 \ ++ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ ++ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 ++ # via ++ # aiosignal ++ # anyio ++ # apache-beam ++ # betterproto ++ # google-cloud-aiplatform ++ # google-genai ++ # opentelemetry-api ++ # opentelemetry-resourcedetector-gcp ++ # opentelemetry-sdk ++ # opentelemetry-semantic-conventions ++ # pydantic ++ # pydantic-core ++ # typing-inspection ++typing-inspection==0.4.2 \ ++ --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ ++ --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 ++ # via pydantic ++urllib3==2.6.3 \ ++ --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ ++ --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 ++ # via requests ++websockets==16.0 \ ++ --hash=sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c \ ++ --hash=sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a \ ++ --hash=sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe \ ++ --hash=sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e \ ++ --hash=sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec \ ++ --hash=sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1 \ ++ --hash=sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64 \ ++ --hash=sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3 \ ++ --hash=sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8 \ ++ --hash=sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206 \ ++ --hash=sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3 \ ++ --hash=sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156 \ ++ --hash=sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d \ ++ --hash=sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9 \ ++ --hash=sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad \ ++ --hash=sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2 \ ++ --hash=sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03 \ ++ --hash=sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8 \ ++ --hash=sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230 \ ++ --hash=sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8 \ ++ --hash=sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea \ ++ --hash=sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641 \ ++ --hash=sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957 \ ++ --hash=sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6 \ ++ --hash=sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6 \ ++ --hash=sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5 \ ++ --hash=sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f \ ++ --hash=sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00 \ ++ --hash=sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e \ ++ --hash=sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b \ ++ --hash=sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72 \ ++ --hash=sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39 \ ++ --hash=sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9 \ ++ --hash=sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79 \ ++ --hash=sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0 \ ++ --hash=sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac \ ++ --hash=sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35 \ ++ --hash=sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0 \ ++ --hash=sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5 \ ++ --hash=sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c \ ++ --hash=sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8 \ ++ --hash=sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1 \ ++ --hash=sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244 \ ++ --hash=sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3 \ ++ --hash=sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767 \ ++ --hash=sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a \ ++ --hash=sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d \ ++ --hash=sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd \ ++ --hash=sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e \ ++ --hash=sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944 \ ++ --hash=sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82 \ ++ --hash=sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d \ ++ --hash=sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4 \ ++ --hash=sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5 \ ++ --hash=sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904 \ ++ --hash=sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde \ ++ --hash=sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f \ ++ --hash=sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c \ ++ --hash=sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89 \ ++ --hash=sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da \ ++ --hash=sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4 ++ # via google-genai ++yarl==1.23.0 \ ++ --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \ ++ --hash=sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4 \ ++ --hash=sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85 \ ++ --hash=sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993 \ ++ --hash=sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222 \ ++ --hash=sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de \ ++ --hash=sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25 \ ++ --hash=sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e \ ++ --hash=sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2 \ ++ --hash=sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e \ ++ --hash=sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860 \ ++ --hash=sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957 \ ++ --hash=sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760 \ ++ --hash=sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52 \ ++ --hash=sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788 \ ++ --hash=sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912 \ ++ --hash=sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719 \ ++ --hash=sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035 \ ++ --hash=sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220 \ ++ --hash=sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412 \ ++ --hash=sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05 \ ++ --hash=sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41 \ ++ --hash=sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 \ ++ --hash=sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4 \ ++ --hash=sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd \ ++ --hash=sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748 \ ++ --hash=sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a \ ++ --hash=sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4 \ ++ --hash=sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34 \ ++ --hash=sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069 \ ++ --hash=sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25 \ ++ --hash=sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2 \ ++ --hash=sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb \ ++ --hash=sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f \ ++ --hash=sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5 \ ++ --hash=sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8 \ ++ --hash=sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c \ ++ --hash=sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512 \ ++ --hash=sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6 \ ++ --hash=sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5 \ ++ --hash=sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9 \ ++ --hash=sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072 \ ++ --hash=sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5 \ ++ --hash=sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277 \ ++ --hash=sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a \ ++ --hash=sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6 \ ++ --hash=sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae \ ++ --hash=sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26 \ ++ --hash=sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2 \ ++ --hash=sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4 \ ++ --hash=sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70 \ ++ --hash=sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723 \ ++ --hash=sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c \ ++ --hash=sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9 \ ++ --hash=sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5 \ ++ --hash=sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e \ ++ --hash=sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c \ ++ --hash=sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4 \ ++ --hash=sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0 \ ++ --hash=sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2 \ ++ --hash=sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b \ ++ --hash=sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7 \ ++ --hash=sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750 \ ++ --hash=sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2 \ ++ --hash=sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474 \ ++ --hash=sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716 \ ++ --hash=sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7 \ ++ --hash=sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123 \ ++ --hash=sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007 \ ++ --hash=sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595 \ ++ --hash=sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe \ ++ --hash=sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea \ ++ --hash=sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598 \ ++ --hash=sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679 \ ++ --hash=sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8 \ ++ --hash=sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83 \ ++ --hash=sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6 \ ++ --hash=sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f \ ++ --hash=sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94 \ ++ --hash=sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51 \ ++ --hash=sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120 \ ++ --hash=sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039 \ ++ --hash=sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1 \ ++ --hash=sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05 \ ++ --hash=sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb \ ++ --hash=sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144 \ ++ --hash=sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa \ ++ --hash=sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a \ ++ --hash=sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99 \ ++ --hash=sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928 \ ++ --hash=sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d \ ++ --hash=sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3 \ ++ --hash=sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434 \ ++ --hash=sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86 \ ++ --hash=sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46 \ ++ --hash=sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319 \ ++ --hash=sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67 \ ++ --hash=sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c \ ++ --hash=sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169 \ ++ --hash=sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c \ ++ --hash=sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59 \ ++ --hash=sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107 \ ++ --hash=sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4 \ ++ --hash=sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a \ ++ --hash=sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb \ ++ --hash=sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f \ ++ --hash=sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769 \ ++ --hash=sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432 \ ++ --hash=sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090 \ ++ --hash=sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764 \ ++ --hash=sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d \ ++ --hash=sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4 \ ++ --hash=sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b \ ++ --hash=sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d \ ++ --hash=sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543 \ ++ --hash=sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24 \ ++ --hash=sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5 \ ++ --hash=sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b \ ++ --hash=sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d \ ++ --hash=sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b \ ++ --hash=sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6 \ ++ --hash=sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735 \ ++ --hash=sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e \ ++ --hash=sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28 \ ++ --hash=sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3 \ ++ --hash=sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401 \ ++ --hash=sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6 \ ++ --hash=sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d ++ # via aiohttp ++zipp==3.23.0 \ ++ --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ ++ --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 ++ # via importlib-metadata ++zstandard==0.25.0 \ ++ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ ++ --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ ++ --hash=sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3 \ ++ --hash=sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f \ ++ --hash=sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6 \ ++ --hash=sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936 \ ++ --hash=sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431 \ ++ --hash=sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250 \ ++ --hash=sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa \ ++ --hash=sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f \ ++ --hash=sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851 \ ++ --hash=sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3 \ ++ --hash=sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9 \ ++ --hash=sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6 \ ++ --hash=sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362 \ ++ --hash=sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649 \ ++ --hash=sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb \ ++ --hash=sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5 \ ++ --hash=sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439 \ ++ --hash=sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137 \ ++ --hash=sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa \ ++ --hash=sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd \ ++ --hash=sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701 \ ++ --hash=sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0 \ ++ --hash=sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043 \ ++ --hash=sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1 \ ++ --hash=sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860 \ ++ --hash=sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611 \ ++ --hash=sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53 \ ++ --hash=sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b \ ++ --hash=sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088 \ ++ --hash=sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e \ ++ --hash=sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa \ ++ --hash=sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2 \ ++ --hash=sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0 \ ++ --hash=sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7 \ ++ --hash=sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf \ ++ --hash=sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388 \ ++ --hash=sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530 \ ++ --hash=sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577 \ ++ --hash=sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902 \ ++ --hash=sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc \ ++ --hash=sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98 \ ++ --hash=sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a \ ++ --hash=sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097 \ ++ --hash=sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea \ ++ --hash=sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09 \ ++ --hash=sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb \ ++ --hash=sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7 \ ++ --hash=sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74 \ ++ --hash=sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b \ ++ --hash=sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b \ ++ --hash=sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b \ ++ --hash=sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91 \ ++ --hash=sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150 \ ++ --hash=sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049 \ ++ --hash=sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27 \ ++ --hash=sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a \ ++ --hash=sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00 \ ++ --hash=sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd \ ++ --hash=sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072 \ ++ --hash=sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c \ ++ --hash=sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c \ ++ --hash=sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065 \ ++ --hash=sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512 \ ++ --hash=sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1 \ ++ --hash=sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f \ ++ --hash=sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2 \ ++ --hash=sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df \ ++ --hash=sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab \ ++ --hash=sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7 \ ++ --hash=sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b \ ++ --hash=sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550 \ ++ --hash=sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0 \ ++ --hash=sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea \ ++ --hash=sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277 \ ++ --hash=sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2 \ ++ --hash=sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7 \ ++ --hash=sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778 \ ++ --hash=sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859 \ ++ --hash=sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d \ ++ --hash=sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751 \ ++ --hash=sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12 \ ++ --hash=sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2 \ ++ --hash=sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d \ ++ --hash=sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0 \ ++ --hash=sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3 \ ++ --hash=sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd \ ++ --hash=sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e \ ++ --hash=sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f \ ++ --hash=sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e \ ++ --hash=sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94 \ ++ --hash=sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708 \ ++ --hash=sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313 \ ++ --hash=sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4 \ ++ --hash=sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c \ ++ --hash=sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344 \ ++ --hash=sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551 \ ++ --hash=sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01 ++ # via apache-beam ++ ++# The following packages are considered to be unsafe in a requirements file: ++setuptools==82.0.0 \ ++ --hash=sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb \ ++ --hash=sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0 ++ # via -r python/default_base_bqmonitor_requirements.txt ++ +diff --git a/python/src/main/python/bigquery-anomaly-detection/setup.py b/python/src/main/python/bigquery-anomaly-detection/setup.py +new file mode 100644 +index 000000000..067561292 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/setup.py +@@ -0,0 +1,14 @@ ++from setuptools import setup, find_packages ++ ++setup( ++ name='bqmonitor', ++ version='0.1.0', ++ description='BigQuery anomaly monitoring pipeline (Dataflow Flex Template)', ++ package_dir={'': 'src'}, ++ packages=find_packages(where='src'), ++ python_requires='>=3.11', ++ install_requires=[ ++ 'apache-beam[gcp]==2.71.0', ++ 'google-cloud-bigquery-storage', ++ ], ++) +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/__init__.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/__init__.py +new file mode 100644 +index 000000000..e69de29bb +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py +new file mode 100644 +index 000000000..88e451d8d +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py +@@ -0,0 +1,1401 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Streaming source for BigQuery change history (APPENDS/CHANGES functions). ++ ++This module provides ``ReadBigQueryChangeHistory``, a streaming PTransform ++that continuously polls BigQuery APPENDS() or CHANGES() functions and emits ++changed rows as an unbounded PCollection. ++ ++**Status: Experimental**: API may change without notice. ++ ++Usage:: ++ ++ import apache_beam as beam ++ from bqmonitor.cdc import ReadBigQueryChangeHistory ++ ++ with beam.Pipeline(options=pipeline_options) as p: ++ changes = ( ++ p ++ | ReadBigQueryChangeHistory( ++ table='my-project:my_dataset.my_table', ++ change_function='APPENDS', ++ poll_interval_sec=60)) ++ ++Architecture: ++ Poll: Polling SDF emits lightweight _QueryRange instructions. ++ Query: _ExecuteQueryFn runs the BQ query, writes to a temp table. ++ Read: SDF reads temp table via Storage Read API with dynamic splitting. ++ Cleanup: Stateful DoFn tracks stream completion, deletes temp tables. ++""" ++ ++import dataclasses ++import datetime ++import logging ++import random ++import sys ++import time ++import uuid ++from typing import Any ++from typing import Dict ++from typing import Iterable ++from typing import List ++from typing import Optional ++from typing import Tuple ++ ++import apache_beam as beam ++from apache_beam.io.gcp import bigquery_tools ++from apache_beam.io.gcp.internal.clients import bigquery ++from apache_beam.io.iobase import WatermarkEstimator ++from apache_beam.io.restriction_trackers import OffsetRange ++from apache_beam.io.restriction_trackers import OffsetRestrictionTracker ++from apache_beam.io.watermark_estimators import ManualWatermarkEstimator ++from apache_beam.metrics import Metrics ++from apache_beam.transforms.core import WatermarkEstimatorProvider ++from apache_beam.transforms import trigger as beam_trigger ++from apache_beam.transforms.window import GlobalWindows ++from apache_beam.transforms.window import TimestampedValue ++from apache_beam.utils import retry ++from apache_beam.utils.timestamp import MAX_TIMESTAMP ++from apache_beam.utils.timestamp import Duration ++from apache_beam.utils.timestamp import Timestamp ++ ++try: ++ from apitools.base.py.exceptions import HttpError ++except ImportError: ++ HttpError = None # type: ignore ++ ++try: ++ from google.cloud import bigquery_storage_v1 as bq_storage ++except ImportError: ++ bq_storage = None # type: ignore ++ ++try: ++ import pyarrow ++except ImportError: ++ pyarrow = None # type: ignore ++ ++_LOGGER = logging.getLogger(__name__) ++ ++__all__ = ['ReadBigQueryChangeHistory'] ++ ++# Max time range for CHANGES() queries: 1 day. ++_MAX_CHANGES_RANGE = Duration(seconds=86400) ++ ++# Side output tag for cleanup signals between the Read SDF and Cleanup DoFn. ++_CLEANUP_TAG = 'cleanup' ++ ++# Default number of Storage Read API streams to request. ++# Matches ReadFromBigQuery's MIN_SPLIT_COUNT to enable parallelism. ++# The server may return fewer streams if the table is small. ++_DEFAULT_MAX_STREAMS = 10 ++ ++# Default table expiration for auto-created temp datasets: 24 hours in ms. ++# Tables created in the dataset auto-expire after this duration if not ++# explicitly deleted, acting as a safety net for orphaned temp tables ++# (e.g. pipeline crash before cleanup runs). ++_DEFAULT_TABLE_EXPIRATION_MS = 24 * 60 * 60 * 1000 ++ ++ ++@dataclasses.dataclass ++class _QueryResult: ++ """Bridges the Query step (query execution) to the Read SDF. ++ ++ After _ExecuteQueryFn runs a CHANGES/APPENDS query, it emits a _QueryResult ++ pointing to the temp table containing query results. The Read SDF reads ++ rows from that temp table via the Storage Read API. ++ ++ range_start/range_end define the time window this query covers as Beam ++ Timestamps (int microseconds internally). The Read SDF uses range_start ++ to set an initial watermark hold so the runner doesn't advance the ++ watermark past the data's timestamps. ++ """ ++ temp_table_ref: 'bigquery.TableReference' ++ range_start: Timestamp ++ range_end: Timestamp ++ ++ ++@dataclasses.dataclass ++class _PollConfig: ++ """Input element for the polling SDF. ++ ++ Only contains start_time (Beam Timestamp), which ++ _PollWatermarkEstimatorProvider uses to initialize the watermark hold. ++ All other config is passed via _PollChangeHistoryFn.__init__. ++ """ ++ start_time: Timestamp ++ ++ ++@dataclasses.dataclass ++class _QueryRange: ++ """Lightweight instruction emitted by the polling SDF. ++ ++ Contains only the time range to query as Beam Timestamps (int microseconds ++ internally). Static config (table, project, etc.) is held by ++ _ExecuteQueryFn which receives these after a Reshuffle commit boundary, ++ preventing duplicate queries on SDF re-dispatch. ++ """ ++ chunk_start: Timestamp ++ chunk_end: Timestamp ++ ++ ++class _StreamRestriction: ++ """Restriction carrying BQ Storage stream names for cross-worker safety. ++ ++ Unlike a plain OffsetRange(0, N), this restriction is self-contained: ++ each split carries the actual stream name strings so it can be processed ++ on any worker. Composes an OffsetRange for offset logic. ++ """ ++ __slots__ = ('stream_names', 'range') ++ ++ def __init__( ++ self, stream_names: Tuple[str, ...], start: int, stop: int) -> None: ++ self.stream_names = stream_names # tuple of BQ stream name strings ++ self.range = OffsetRange(start, stop) ++ ++ @property ++ def start(self) -> int: ++ return self.range.start ++ ++ @property ++ def stop(self) -> int: ++ return self.range.stop ++ ++ def __eq__(self, other: object) -> bool: ++ if not isinstance(other, _StreamRestriction): ++ return False ++ return ( ++ self.stream_names == other.stream_names and self.range == other.range) ++ ++ def __hash__(self) -> int: ++ return hash((type(self), self.stream_names, self.range)) ++ ++ def __repr__(self) -> str: ++ return ( ++ '_StreamRestriction(streams=%d, start=%d, stop=%d)' % ++ (len(self.stream_names), self.start, self.stop)) ++ ++ def size(self) -> int: ++ return self.range.size() ++ ++ ++class _StreamRestrictionTracker(beam.io.iobase.RestrictionTracker): ++ """Tracker for _StreamRestriction, delegating offset logic to ++ OffsetRestrictionTracker.""" ++ def __init__(self, restriction: _StreamRestriction) -> None: ++ self._stream_names = restriction.stream_names ++ self._offset_tracker = OffsetRestrictionTracker(restriction.range) ++ ++ def current_restriction(self) -> _StreamRestriction: ++ r = self._offset_tracker.current_restriction() ++ return _StreamRestriction(self._stream_names, r.start, r.stop) ++ ++ def try_claim(self, position: int) -> bool: ++ return self._offset_tracker.try_claim(position) ++ ++ def try_split( ++ self, fraction_of_remainder: float ++ ) -> Optional[Tuple[_StreamRestriction, _StreamRestriction]]: ++ result = self._offset_tracker.try_split(fraction_of_remainder) ++ if result is not None: ++ primary, residual = result ++ return ( ++ _StreamRestriction(self._stream_names, primary.start, primary.stop), ++ _StreamRestriction(self._stream_names, residual.start, residual.stop)) ++ return None ++ ++ def check_done(self) -> None: ++ self._offset_tracker.check_done() ++ ++ def current_progress(self): ++ return self._offset_tracker.current_progress() ++ ++ def is_bounded(self) -> bool: ++ return True ++ ++ ++class _NonSplittableOffsetTracker(OffsetRestrictionTracker): ++ """OffsetRestrictionTracker that allows checkpointing but prevents splitting. ++ ++ Checkpointing (fraction=0) is required for defer_remainder(). All other ++ split fractions are refused, ensuring the polling SDF runs as a singleton. ++ """ ++ def try_split( ++ self, fraction_of_remainder: float ++ ) -> Optional[Tuple[OffsetRange, OffsetRange]]: ++ if fraction_of_remainder == 0: ++ return super().try_split(fraction_of_remainder) ++ return None ++ ++ ++class _PollWatermarkEstimator(WatermarkEstimator): ++ """Watermark estimator that tracks both a watermark hold and poll cursor. ++ ++ The watermark hold (reported via current_watermark) is set to start_ts: ++ the earliest data timestamp emitted by the current poll. This prevents ++ downstream stages from seeing data as late. ++ ++ The poll cursor (last_end) tracks where the next poll should start. ++ This is separate from the watermark so we can hold the watermark back ++ at start_ts while still advancing the poll cursor to end_ts. ++ ++ All timestamps are Beam Timestamps (int microseconds internally). ++ ++ State is checkpointed as (watermark_hold, last_end) so ++ both values survive SDF re-dispatch. ++ """ ++ def __init__(self, state: Tuple[Timestamp, Timestamp]) -> None: ++ self._watermark_hold, self._last_end = state ++ ++ def observe_timestamp(self, timestamp: Timestamp) -> None: ++ pass ++ ++ def current_watermark(self) -> Timestamp: ++ return self._watermark_hold ++ ++ def get_estimator_state(self) -> Tuple[Timestamp, Timestamp]: ++ return (self._watermark_hold, self._last_end) ++ ++ def set_watermark(self, timestamp: Timestamp) -> None: ++ if not isinstance(timestamp, Timestamp): ++ raise ValueError('set_watermark expects a Timestamp as input') ++ if self._watermark_hold and self._watermark_hold > timestamp: ++ raise ValueError( ++ 'Watermark must be monotonically increasing. ' ++ 'Provided %s < current %s' % (timestamp, self._watermark_hold)) ++ self._watermark_hold = timestamp ++ ++ def advance_poll_cursor(self, end: Timestamp) -> None: ++ """Record end so the next poll starts from here. ++ ++ Only advances forward: if end is earlier than the current cursor ++ (e.g. BQ clock regression), the cursor stays put so the next poll ++ doesn't re-query an already-covered range. ++ """ ++ self._last_end = max(self._last_end, end) ++ ++ def poll_cursor(self) -> Timestamp: ++ """Return the start Timestamp for the next poll.""" ++ return self._last_end ++ ++ ++class _PollWatermarkEstimatorProvider(WatermarkEstimatorProvider): ++ """Provider for _PollWatermarkEstimator. ++ ++ Initializes with watermark hold at start_time and poll cursor at ++ start_time (first poll will query from start_time). ++ """ ++ def initial_estimator_state( ++ self, element: _PollConfig, ++ restriction: OffsetRange) -> Tuple[Timestamp, Timestamp]: ++ return (element.start_time, element.start_time) ++ ++ def create_watermark_estimator( ++ self, estimator_state: Tuple[Timestamp, ++ Timestamp]) -> _PollWatermarkEstimator: ++ return _PollWatermarkEstimator(estimator_state) ++ ++ ++def _table_key(table_ref: 'bigquery.TableReference') -> str: ++ """Convert a TableReference to a 'project.dataset.table' string.""" ++ return f'{table_ref.projectId}.{table_ref.datasetId}.{table_ref.tableId}' ++ ++ ++def build_changes_query( ++ table: str, ++ start: Timestamp, ++ end: Timestamp, ++ change_function: str, ++ change_type_column: str = 'change_type', ++ change_timestamp_column: str = 'change_timestamp', ++ columns: Optional[List[str]] = None, ++ row_filter: Optional[str] = None) -> str: ++ """Build a CHANGES() or APPENDS() SQL query. ++ ++ Args: ++ table: Table name as 'project.dataset.table' or 'project:dataset.table'. ++ start: Start timestamp (Beam Timestamp). Inclusive. ++ end: End timestamp (Beam Timestamp). Exclusive. ++ change_function: 'CHANGES' or 'APPENDS'. ++ change_type_column: Output column name for _CHANGE_TYPE pseudo-column. ++ change_timestamp_column: Output column name for _CHANGE_TIMESTAMP ++ pseudo-column. ++ columns: Optional list of column names to select. If None, selects all ++ columns. Pseudo-columns are always appended regardless. ++ row_filter: Optional SQL WHERE clause (without the WHERE keyword). ++ Applied after the CHANGES/APPENDS function. ++ ++ Returns: ++ SQL string. ++ """ ++ # Normalize 'project:dataset.table' to 'project.dataset.table' ++ table = table.replace(':', '.') ++ start_iso = start.to_rfc3339() ++ end_iso = end.to_rfc3339() ++ # Pseudo-columns (_CHANGE_TYPE, _CHANGE_TIMESTAMP) can't be written to ++ # destination tables with their original names. Rename them so they can ++ # be persisted to the temp table for Storage Read API reading. ++ pseudo = ( ++ f"_CHANGE_TYPE AS {change_type_column}, " ++ f"_CHANGE_TIMESTAMP AS {change_timestamp_column}") ++ if columns is None: ++ select = f"SELECT * EXCEPT(_CHANGE_TYPE, _CHANGE_TIMESTAMP), {pseudo}" ++ else: ++ select = f"SELECT {', '.join(columns)}, {pseudo}" ++ from_clause = ( ++ f"FROM {change_function}" ++ f"(TABLE `{table}`, " ++ f"TIMESTAMP '{start_iso}', " ++ f"TIMESTAMP '{end_iso}')") ++ where = f" WHERE {row_filter}" if row_filter else "" ++ return f"{select} {from_clause}{where}" ++ ++ ++def compute_ranges(start: Timestamp, end: Timestamp, ++ change_function: str) -> List[Tuple[Timestamp, Timestamp]]: ++ """Split [start, end) into query-safe chunks. ++ ++ CHANGES() has a max 1-day range. APPENDS() has no limit. ++ ++ Args: ++ start: Start Timestamp. Inclusive. ++ end: End Timestamp. Exclusive. ++ change_function: 'CHANGES' or 'APPENDS'. ++ ++ Returns: ++ List of (start, end) Timestamp tuples. Empty if end <= start. ++ """ ++ if end <= start: ++ return [] ++ ++ if change_function != 'CHANGES': ++ return [(start, end)] ++ ++ # CHANGES: chunk into <=1-day ranges ++ ranges = [] ++ current = start ++ while current < end: ++ chunk_end = min(current + _MAX_CHANGES_RANGE, end) ++ ranges.append((current, chunk_end)) ++ current = chunk_end ++ return ranges ++ ++ ++def _utc(ts: Timestamp) -> str: ++ """Format a Beam Timestamp as a concise UTC string for logging.""" ++ return ts.to_utc_datetime(has_tz=True).strftime('%Y-%m-%dT%H:%M:%S.%f') ++ ++ ++# ============================================================================= ++# Poll: _PollChangeHistoryFn (Polling SDF) ++# ============================================================================= ++ ++ ++class _PollChangeHistoryFn(beam.DoFn, beam.transforms.core.RestrictionProvider): ++ """SDF that periodically emits _QueryRange instructions. ++ ++ Uses defer_remainder() for poll timing and _PollWatermarkEstimator to ++ control the watermark. The watermark is initially held at start_time, then ++ advanced to start_ts of each poll. ++ ++ All timestamps are Beam Timestamps (int microseconds internally). ++ Durations (buffer, poll_interval) are Beam Durations. ++ ++ Derives start_ts from the poll cursor. On each poll: ++ 1. start_ts = poll cursor (last end_ts, or start_time on first poll) ++ 2. end_ts = bq_now - buffer ++ 3. Computes query chunks, yields _QueryRange per chunk ++ 4. Advances poll cursor to end_ts (for next poll's start) ++ 5. Advances watermark to start_ts (earliest data in this poll) ++ 6. Defers to next poll interval ++ """ ++ def __init__( ++ self, ++ table: str, ++ project: str, ++ change_function: str, ++ buffer: Duration, ++ start_time: Timestamp, ++ stop_time: Timestamp, ++ poll_interval: Duration, ++ location: Optional[str] = None) -> None: ++ self._table = table ++ self._project = project ++ self._change_function = change_function ++ self._buffer = buffer ++ self._start_time = start_time ++ self._stop_time = stop_time ++ self._poll_interval = poll_interval ++ self._location = location ++ ++ def setup(self) -> None: ++ self._bq_wrapper = bigquery_tools.BigQueryWrapper() ++ if self._location is None: ++ table_ref = bigquery_tools.parse_table_reference( ++ self._table, project=self._project) ++ self._location = self._bq_wrapper.get_table_location( ++ table_ref.projectId, table_ref.datasetId, table_ref.tableId) ++ _LOGGER.info( ++ '[Poll] Inferred location=%s from source table %s', ++ self._location, ++ self._table) ++ ++ @retry.with_exponential_backoff( ++ num_retries=3, ++ retry_filter=retry.retry_on_server_errors_and_timeout_filter) ++ def _get_bq_timestamp(self) -> Timestamp: ++ """Query BigQuery for the current server timestamp. ++ ++ Returns a Beam Timestamp created from integer microseconds. ++ Uses BQ's CURRENT_TIMESTAMP instead of the local clock to avoid ++ data loss from clock skew between the worker VM and BigQuery. ++ """ ++ request = bigquery.BigqueryJobsQueryRequest( ++ projectId=self._project, ++ queryRequest=bigquery.QueryRequest( ++ query='SELECT UNIX_MICROS(CURRENT_TIMESTAMP()) AS ts', ++ useLegacySql=False, ++ location=self._location)) ++ response = self._bq_wrapper.client.jobs.Query(request) ++ return Timestamp(micros=int(response.rows[0].f[0].v.string_value)) ++ ++ def initial_restriction(self, element: _PollConfig) -> OffsetRange: ++ return OffsetRange(0, sys.maxsize) ++ ++ def create_tracker( ++ self, restriction: OffsetRange) -> _NonSplittableOffsetTracker: ++ # Guarantee at least one poll cycle: restriction.start == 0 on the first ++ # invocation (from initial_restriction). After the first try_claim(0) + ++ # defer_remainder, subsequent invocations arrive with start >= 1. ++ if restriction.start > 0 and time.time() >= float(self._stop_time): ++ _LOGGER.info( ++ '[Poll] create_tracker: stop_time reached, ' ++ 'returning empty range to terminate SDF') ++ return _NonSplittableOffsetTracker( ++ OffsetRange(restriction.start, restriction.start)) ++ return _NonSplittableOffsetTracker(restriction) ++ ++ def restriction_size( ++ self, element: _PollConfig, restriction: OffsetRange) -> int: ++ return 1 ++ ++ def split(self, element: _PollConfig, ++ restriction: OffsetRange) -> Iterable[OffsetRange]: ++ yield restriction ++ ++ def truncate(self, element: _PollConfig, restriction: OffsetRange) -> None: ++ return None ++ ++ def _next_poll_time(self, start_ts: Timestamp, ++ now: float) -> Optional[Timestamp]: ++ """Return a Timestamp to defer to, or None if we should poll now.""" ++ earliest = start_ts + self._buffer + self._poll_interval ++ if now < float(earliest): ++ return earliest ++ return None ++ ++ def _emit_query_ranges( ++ self, ++ start_ts: Timestamp, ++ end_ts: Timestamp, ++ watermark_estimator: _PollWatermarkEstimator) -> Iterable[_QueryRange]: ++ """Compute and yield _QueryRange elements, advancing estimator state.""" ++ ranges = compute_ranges(start_ts, end_ts, self._change_function) ++ _LOGGER.info( ++ '[Poll] %d chunks for [%s, %s)', ++ len(ranges), ++ _utc(start_ts), ++ _utc(end_ts)) ++ Metrics.counter('BigQueryChangeHistory', 'polls').inc() ++ ++ watermark_estimator.advance_poll_cursor(end_ts) ++ watermark_estimator.set_watermark(start_ts) ++ _LOGGER.info( ++ '[Poll] Watermark=%s (start_ts), cursor=%s (end_ts)', ++ _utc(start_ts), ++ _utc(end_ts)) ++ ++ for chunk_start, chunk_end in ranges: ++ yield TimestampedValue( ++ _QueryRange(chunk_start=chunk_start, chunk_end=chunk_end), start_ts) ++ ++ @beam.DoFn.unbounded_per_element() ++ def process( ++ self, ++ _: _PollConfig, ++ restriction_tracker=beam.DoFn.RestrictionParam(), ++ watermark_estimator=beam.DoFn.WatermarkEstimatorParam( ++ _PollWatermarkEstimatorProvider()) ++ ) -> Iterable[_QueryRange]: ++ ++ now = time.time() ++ start_ts = watermark_estimator.poll_cursor() ++ ++ defer_to = self._next_poll_time(start_ts, now) ++ if defer_to is not None: ++ restriction_tracker.defer_remainder(defer_to) ++ return ++ ++ # Use BQ server time instead of local clock to avoid data loss ++ # from clock skew between the worker VM and BigQuery. ++ bq_now = self._get_bq_timestamp() ++ end_ts = min(bq_now - self._buffer, self._stop_time) ++ ++ _LOGGER.info( ++ '[Poll] Polling: start=%s, end=%s, watermark=%s, ' ++ 'clock_skew=%.3fs', ++ _utc(start_ts), ++ _utc(end_ts), ++ _utc(watermark_estimator.current_watermark()), ++ float(bq_now) - now) ++ ++ current_index = restriction_tracker.current_restriction().start ++ ++ if not restriction_tracker.try_claim(current_index): ++ return ++ restriction_tracker.defer_remainder(Timestamp.of(now) + self._poll_interval) ++ ++ yield from self._emit_query_ranges(start_ts, end_ts, watermark_estimator) ++ ++ ++class _ExecuteQueryFn(beam.DoFn): ++ """Executes a BQ CHANGES/APPENDS query from a _QueryRange instruction. ++ """ ++ def __init__( ++ self, ++ table: str, ++ project: str, ++ change_function: str, ++ temp_dataset: str, ++ location: Optional[str], ++ change_type_column: str = 'change_type', ++ change_timestamp_column: str = 'change_timestamp', ++ columns: Optional[List[str]] = None, ++ row_filter: Optional[str] = None) -> None: ++ self._table = table ++ self._project = project ++ self._change_function = change_function ++ self._temp_dataset = temp_dataset ++ self._location = location ++ self._change_type_column = change_type_column ++ self._change_timestamp_column = change_timestamp_column ++ self._columns = columns ++ self._row_filter = row_filter ++ ++ def setup(self) -> None: ++ self._bq_wrapper = bigquery_tools.BigQueryWrapper() ++ if self._location is None: ++ table_ref = bigquery_tools.parse_table_reference( ++ self._table, project=self._project) ++ self._location = self._bq_wrapper.get_table_location( ++ table_ref.projectId, table_ref.datasetId, table_ref.tableId) ++ _LOGGER.info( ++ '[Query] Inferred location=%s from source table %s', ++ self._location, ++ self._table) ++ self._get_or_create_temp_dataset() ++ ++ def _get_or_create_temp_dataset(self) -> None: ++ """Create the temp dataset if it doesn't exist. ++ ++ Sets a default table expiration so orphaned temp tables (e.g. from ++ pipeline crashes before cleanup) are automatically garbage-collected. ++ """ ++ try: ++ self._bq_wrapper.client.datasets.Get( ++ bigquery.BigqueryDatasetsGetRequest( ++ projectId=self._project, datasetId=self._temp_dataset)) ++ except HttpError as e: ++ if e.status_code != 404: ++ raise ++ _LOGGER.info( ++ '[Query] Creating temp dataset %s:%s (location=%s)', ++ self._project, self._temp_dataset, self._location) ++ dataset_ref = bigquery.DatasetReference( ++ projectId=self._project, datasetId=self._temp_dataset) ++ dataset = bigquery.Dataset( ++ datasetReference=dataset_ref, ++ defaultTableExpirationMs=_DEFAULT_TABLE_EXPIRATION_MS) ++ if self._location is not None: ++ dataset.location = self._location ++ self._bq_wrapper.client.datasets.Insert( ++ bigquery.BigqueryDatasetsInsertRequest( ++ projectId=self._project, dataset=dataset)) ++ ++ def process(self, qr: _QueryRange) -> Iterable[_QueryResult]: ++ """Execute the BQ query described by a _QueryRange and yield _QueryResult. ++ """ ++ ++ sql = build_changes_query( ++ self._table, ++ qr.chunk_start, ++ qr.chunk_end, ++ self._change_function, ++ self._change_type_column, ++ self._change_timestamp_column, ++ self._columns, ++ self._row_filter) ++ temp_table_id = f'beam_ch_temp_{uuid.uuid4().hex[:8]}' ++ job_id = f'beam_ch_{uuid.uuid4().hex[:12]}' ++ ++ _LOGGER.info( ++ '[Query] job_id=%s, temp_table=%s.%s, range=[%s, %s)', ++ job_id, ++ self._temp_dataset, ++ temp_table_id, ++ _utc(qr.chunk_start), ++ _utc(qr.chunk_end)) ++ ++ temp_table_ref = bigquery.TableReference( ++ projectId=self._project, ++ datasetId=self._temp_dataset, ++ tableId=temp_table_id) ++ ++ reference = bigquery.JobReference( ++ jobId=job_id, projectId=self._project, location=self._location) ++ ++ request = bigquery.BigqueryJobsInsertRequest( ++ projectId=self._project, ++ job=bigquery.Job( ++ configuration=bigquery.JobConfiguration( ++ query=bigquery.JobConfigurationQuery( ++ query=sql, ++ useLegacySql=False, ++ destinationTable=temp_table_ref, ++ writeDisposition='WRITE_TRUNCATE', ++ ), ++ ), ++ jobReference=reference)) ++ ++ _LOGGER.info('[Query] Submitting BQ job %s...', job_id) ++ response = self._bq_wrapper._start_job(request) ++ _LOGGER.info('[Query] BQ job %s submitted, waiting...', job_id) ++ self._bq_wrapper.wait_for_bq_job( ++ response.jobReference, sleep_duration_sec=2) ++ _LOGGER.info( ++ '[Query] BQ job %s DONE. Results in %s.%s', ++ job_id, ++ self._temp_dataset, ++ temp_table_id) ++ Metrics.counter('BigQueryChangeHistory', 'queries').inc() ++ ++ yield _QueryResult( ++ temp_table_ref=temp_table_ref, ++ range_start=qr.chunk_start, ++ range_end=qr.chunk_end) ++ ++ ++class _CDCWatermarkEstimatorProvider(WatermarkEstimatorProvider): ++ """WatermarkEstimatorProvider that initializes the hold from _QueryResult. ++ ++ Uses range_start from the element to set the initial watermark hold. ++ This prevents the runner from advancing the watermark past the data's ++ timestamps before any rows are emitted. ++ """ ++ def initial_estimator_state( ++ self, element: _QueryResult, ++ restriction: _StreamRestriction) -> Timestamp: ++ return element.range_start ++ ++ def create_watermark_estimator( ++ self, estimator_state: Timestamp) -> ManualWatermarkEstimator: ++ return ManualWatermarkEstimator(estimator_state) ++ ++ ++# ============================================================================= ++# Read: _ReadStorageStreamsSDF ++# ============================================================================= ++ ++ ++class _ReadStorageStreamsSDF(beam.DoFn, ++ beam.transforms.core.RestrictionProvider): ++ """SDF that reads a temp table via BigQuery Storage Read API. ++ ++ Note on SDF lifecycle: the runner decomposes this SDF into three internal ++ wrapper DoFns, each a separately deserialized copy: ++ - Stage A (PairWithRestriction): calls initial_restriction(): no setup() ++ - Stage B (SplitAndSizeRestrictions): calls split(), restriction_size() ++ - Stage C (ProcessSizedElements): calls setup(), then process() ++ Because initial_restriction() runs on a different copy than process(), ++ _ensure_client() lazily creates a gRPC client on whichever copy needs one. ++ The _StreamRestriction carries stream names directly so no shared state ++ is needed between copies. ++ ++ Each element is a _QueryResult pointing to a temp table. ++ ++ Watermark: Uses ManualWatermarkEstimator so the watermark only advances ++ as fast as the change-timestamp values we emit. ++ ++ Emits: ++ Main output: TimestampedValue(row_dict, event_timestamp) ++ Side output (_CLEANUP_TAG): (table_key, (streams_read, total_streams)) ++ """ ++ def __init__( ++ self, ++ batch_arrow_read: bool = True, ++ change_timestamp_column: str = 'change_timestamp', ++ max_split_rounds: int = 1, ++ emit_raw_batches: bool = False) -> None: ++ self._batch_arrow_read = batch_arrow_read ++ self._change_timestamp_column = change_timestamp_column ++ self._max_split_rounds = max_split_rounds ++ self._emit_raw_batches = emit_raw_batches ++ self._storage_client = None ++ ++ def _ensure_client(self) -> None: ++ """Lazily initialize the Storage client. ++ ++ Called from both setup() and initial_restriction() because the runner ++ may invoke initial_restriction on the RestrictionProvider instance ++ before setup() runs (or on a separately deserialized copy). ++ """ ++ if self._storage_client is None: ++ _LOGGER.info('[Read] creating BigQueryReadClient') ++ self._storage_client = bq_storage.BigQueryReadClient() ++ ++ def setup(self) -> None: ++ self._ensure_client() ++ ++ def _split_all_streams(self, stream_names: Tuple[str, ...], ++ max_split_rounds: int) -> Tuple[str, ...]: ++ """Split each stream at fraction=0.5 for up to max_split_rounds rounds. ++ ++ Each round attempts to split every stream in the current list. A ++ successful split replaces the original stream with primary + remainder. ++ A refused split (both fields empty) keeps the original stream intact. ++ Stops when max_split_rounds is reached or a full round produces zero ++ new splits. ++ ++ BQ's server-side granularity controls how many splits are possible. ++ Small tables may not split at all; large tables may allow multiple ++ rounds of doubling. ++ """ ++ result = list(stream_names) ++ for round_num in range(1, max_split_rounds + 1): ++ new_result = [] ++ made_progress = False ++ for name in result: ++ response = self._storage_client.split_read_stream( ++ request=bq_storage.types.SplitReadStreamRequest( ++ name=name, fraction=0.5)) ++ primary = response.primary_stream.name ++ remainder = response.remainder_stream.name ++ if primary and remainder: ++ new_result.extend([primary, remainder]) ++ made_progress = True ++ else: ++ new_result.append(name) ++ result = new_result ++ _LOGGER.info( ++ '[Read] _split_all_streams round %d/%d: %d streams ' ++ '(progress=%s)', ++ round_num, ++ max_split_rounds, ++ len(result), ++ made_progress) ++ if not made_progress: ++ break ++ return tuple(result) ++ ++ def initial_restriction(self, element: _QueryResult) -> _StreamRestriction: ++ """Create ReadSession and return _StreamRestriction with stream names. ++ ++ When max_split_rounds > 0, uses SplitReadStream to subdivide each ++ stream at fraction=0.5 for up to max_split_rounds rounds, maximizing ++ parallelism beyond what CreateReadSession provides. ++ """ ++ self._ensure_client() ++ table_key = _table_key(element.temp_table_ref) ++ session = self._create_read_session(element.temp_table_ref) ++ stream_names = tuple(s.name for s in session.streams) ++ original_count = len(stream_names) ++ _LOGGER.info( ++ '[Read] initial_restriction for %s: %d streams from CreateReadSession', ++ table_key, ++ original_count) ++ ++ if self._max_split_rounds > 0: ++ stream_names = self._split_all_streams( ++ stream_names, self._max_split_rounds) ++ _LOGGER.info( ++ '[Read] initial_restriction for %s: %d -> %d streams ' ++ 'after SplitReadStream', ++ table_key, ++ original_count, ++ len(stream_names)) ++ ++ return _StreamRestriction(stream_names, 0, len(stream_names)) ++ ++ def create_tracker( ++ self, restriction: _StreamRestriction) -> _StreamRestrictionTracker: ++ return _StreamRestrictionTracker(restriction) ++ ++ def restriction_size( ++ self, element: _QueryResult, restriction: _StreamRestriction) -> int: ++ return restriction.size() ++ ++ def split(self, element: _QueryResult, ++ restriction: _StreamRestriction) -> Iterable[_StreamRestriction]: ++ """Yield one _StreamRestriction per stream for parallel distribution.""" ++ if restriction.size() <= 1: ++ yield restriction ++ else: ++ for i in range(restriction.start, restriction.stop): ++ yield _StreamRestriction(restriction.stream_names, i, i + 1) ++ ++ def is_bounded(self) -> bool: ++ return True ++ ++ def process( ++ self, ++ element: _QueryResult, ++ restriction_tracker=beam.DoFn.RestrictionParam(), ++ watermark_estimator=beam.DoFn.WatermarkEstimatorParam( ++ _CDCWatermarkEstimatorProvider()) ++ ): ++ self._ensure_client() ++ table_key = _table_key(element.temp_table_ref) ++ ++ _LOGGER.info( ++ '[Read] Processing %s, range=[%s, %s), ' ++ 'initial watermark=%s', ++ table_key, ++ _utc(element.range_start), ++ _utc(element.range_end), ++ _utc(watermark_estimator.current_watermark())) ++ ++ restriction = restriction_tracker.current_restriction() ++ stream_names = restriction.stream_names ++ total_streams = len(stream_names) ++ ++ streams_read = 0 ++ ++ _LOGGER.info( ++ '[Read] Reading streams [%d, %d) of %d total for %s', ++ restriction.start, ++ restriction.stop, ++ total_streams, ++ table_key) ++ ++ for i in range(restriction.start, restriction.stop): ++ if not restriction_tracker.try_claim(i): ++ _LOGGER.info( ++ '[Read] try_claim(%d) FAILED for %s: ' ++ 'runner split or checkpoint, breaking', ++ i, ++ table_key) ++ break ++ ++ stream_name = stream_names[i] ++ _LOGGER.info( ++ '[Read] try_claim(%d) succeeded: reading stream %s', i, stream_name) ++ ++ stream_rows = 0 ++ if self._emit_raw_batches: ++ stream_batches = 0 ++ for raw_batch in self._read_stream_raw(stream_name): ++ yield TimestampedValue(raw_batch, element.range_start) ++ stream_batches += 1 ++ Metrics.counter( ++ 'BigQueryChangeHistory', 'batches_emitted').inc(stream_batches) ++ else: ++ for row in self._read_stream(stream_name): ++ ts = row.get(self._change_timestamp_column) ++ if ts is None: ++ raise ValueError( ++ 'Row missing %r column. Row keys: %s' % ++ (self._change_timestamp_column, list(row.keys()))) ++ if isinstance(ts, datetime.datetime): ++ ts = Timestamp.from_utc_datetime(ts) ++ ++ yield TimestampedValue(row, ts) ++ stream_rows += 1 ++ Metrics.counter( ++ 'BigQueryChangeHistory', 'rows_emitted').inc(stream_rows) ++ ++ streams_read += 1 ++ _LOGGER.info( ++ '[Read] Finished reading stream %d for %s: %d rows', ++ i, ++ table_key, ++ stream_rows) ++ Metrics.counter('BigQueryChangeHistory', 'streams_read').inc() ++ ++ # Advance watermark to range_end after reading all streams. The ++ # initial hold was set to range_start by _CDCWatermarkEstimatorProvider. ++ watermark_estimator.set_watermark(element.range_end) ++ _LOGGER.info( ++ '[Read] Watermark advanced to %s (range_end) for %s', ++ _utc(element.range_end), ++ table_key) ++ ++ # Release the storage client so the gRPC channel doesn't go stale ++ # between process() calls. _ensure_client() will create a fresh one. ++ self._storage_client = None ++ ++ # Emit cleanup signal. Every split that reads at least one stream ++ # reports how many it read. ++ if streams_read > 0: ++ _LOGGER.info( ++ '[Read] Emitting cleanup signal for %s: ' ++ 'streams_read=%d, total_streams=%d', ++ table_key, ++ streams_read, ++ total_streams) ++ yield beam.pvalue.TaggedOutput( ++ _CLEANUP_TAG, (table_key, (streams_read, total_streams))) ++ ++ def _create_read_session(self, table_ref: 'bigquery.TableReference') -> Any: ++ """Create a BigQuery Storage ReadSession for the given table.""" ++ table_path = ( ++ f'projects/{table_ref.projectId}/' ++ f'datasets/{table_ref.datasetId}/' ++ f'tables/{table_ref.tableId}') ++ ++ requested_session = bq_storage.types.ReadSession() ++ requested_session.table = table_path ++ requested_session.data_format = bq_storage.types.DataFormat.ARROW ++ read_options = requested_session.read_options ++ read_options.arrow_serialization_options.buffer_compression = ( ++ bq_storage.types.ArrowSerializationOptions.CompressionCodec.ZSTD) ++ ++ session = self._storage_client.create_read_session( ++ parent=f'projects/{table_ref.projectId}', ++ read_session=requested_session, ++ max_stream_count=_DEFAULT_MAX_STREAMS) ++ _LOGGER.info( ++ '[Read] _create_read_session: table=%s, %d streams', ++ table_path, ++ len(session.streams)) ++ return session ++ ++ def _read_stream(self, stream_name: str) -> Iterable[Dict[str, Any]]: ++ """Read all rows from a single Storage API stream as dicts. ++ ++ When batch_arrow_read is enabled, converts entire Arrow RecordBatches ++ at once using to_pylist() instead of calling .as_py() on each cell ++ individually. This is ~1.5x faster for large tables at the cost of ~2x ++ peak memory per batch. ++ """ ++ if self._batch_arrow_read: ++ yield from self._read_stream_batch(stream_name) ++ else: ++ yield from self._read_stream_row_by_row(stream_name) ++ ++ def _read_stream_row_by_row(self, ++ stream_name: str) -> Iterable[Dict[str, Any]]: ++ """Row-by-row Arrow conversion (lower memory than batch mode).""" ++ t0 = time.time() ++ row_count = 0 ++ for row in self._storage_client.read_rows(stream_name).rows(): ++ yield dict((item[0], item[1].as_py()) for item in row.items()) ++ row_count += 1 ++ elapsed = time.time() - t0 ++ _LOGGER.info( ++ '[Read] row_by_row: %d rows in %.2fs (%.0f rows/s)', ++ row_count, ++ elapsed, ++ row_count / elapsed if elapsed > 0 else 0) ++ ++ def _read_stream_batch(self, stream_name: str) -> Iterable[Dict[str, Any]]: ++ """Batch-convert Arrow RecordBatches for high throughput.""" ++ schema = None ++ row_count = 0 ++ t0 = time.time() ++ for response in self._storage_client.read_rows(stream_name): ++ if schema is None and response.arrow_schema.serialized_schema: ++ schema = pyarrow.ipc.read_schema( ++ pyarrow.py_buffer(response.arrow_schema.serialized_schema)) ++ batch_bytes = response.arrow_record_batch.serialized_record_batch ++ if batch_bytes and schema is not None: ++ batch = pyarrow.ipc.read_record_batch( ++ pyarrow.py_buffer(batch_bytes), schema) ++ yield from batch.to_pylist() ++ row_count += batch.num_rows ++ elapsed = time.time() - t0 ++ _LOGGER.info( ++ '[Read] batch_read: %d rows in %.2fs (%.0f rows/s)', ++ row_count, ++ elapsed, ++ row_count / elapsed if elapsed > 0 else 0) ++ ++ def _read_stream_raw( ++ self, ++ stream_name: str) -> Iterable[Tuple[bytes, bytes]]: ++ """Yield raw (schema_bytes, batch_bytes) without decompression. ++ ++ Used when emit_raw_batches is enabled to defer decompression and ++ Arrow-to-Python conversion to a downstream DoFn after reshuffling. ++ Schema bytes are included in each tuple so each batch is ++ self-contained and can be decoded independently. ++ """ ++ schema_bytes = b'' ++ batch_count = 0 ++ t0 = time.time() ++ for response in self._storage_client.read_rows(stream_name): ++ if not schema_bytes and response.arrow_schema.serialized_schema: ++ schema_bytes = bytes(response.arrow_schema.serialized_schema) ++ batch_bytes = response.arrow_record_batch.serialized_record_batch ++ if batch_bytes and schema_bytes: ++ yield (schema_bytes, bytes(batch_bytes)) ++ batch_count += 1 ++ elapsed = time.time() - t0 ++ _LOGGER.info( ++ '[Read] raw_read: %d batches in %.2fs', ++ batch_count, ++ elapsed) ++ ++ ++class _DecompressArrowBatchesFn(beam.DoFn): ++ """Decompress and convert raw Arrow batches to timestamped row dicts. ++ ++ Receives GBK output: (shard_key, Iterable[(schema_bytes, batch_bytes)]) ++ and converts each batch to individual row dicts with event timestamps ++ extracted from the change_timestamp column. ++ """ ++ def __init__(self, change_timestamp_column: str = 'change_timestamp') -> None: ++ self._change_timestamp_column = change_timestamp_column ++ ++ def process( ++ self, ++ element: Tuple[int, Iterable[Tuple[bytes, bytes]]] ++ ) -> Iterable[Dict[str, Any]]: ++ _, batches = element ++ for schema_bytes, batch_bytes in batches: ++ schema = pyarrow.ipc.read_schema(pyarrow.py_buffer(schema_bytes)) ++ batch = pyarrow.ipc.read_record_batch( ++ pyarrow.py_buffer(batch_bytes), schema) ++ ++ rows = batch.to_pylist() ++ for row in rows: ++ ts = row.get(self._change_timestamp_column) ++ if ts is None: ++ raise ValueError( ++ 'Row missing %r column. Row keys: %s' % ++ (self._change_timestamp_column, list(row.keys()))) ++ if isinstance(ts, datetime.datetime): ++ ts = Timestamp.from_utc_datetime(ts) ++ yield TimestampedValue(row, ts) ++ Metrics.counter('BigQueryChangeHistory', 'rows_emitted').inc(len(rows)) ++ ++ ++# ============================================================================= ++# Cleanup: _CleanupTempTablesFn ++# ============================================================================= ++ ++ ++class _CleanupTempTablesFn(beam.DoFn): ++ """Stateful DoFn that deletes temp tables after all streams are read. ++ ++ Receives cleanup signals from the Read SDF as: ++ (table_key, (streams_read_count, total_streams)) ++ ++ Accumulates streams_read across all signals for the same table_key. ++ When streams_read >= total_streams, deletes the temp table. The >= ++ (rather than ==) guards against duplicate delivery in at-least-once runners. ++ """ ++ STREAMS_READ = beam.transforms.userstate.CombiningValueStateSpec( ++ 'streams_read', sum) ++ ++ def setup(self) -> None: ++ _LOGGER.info('[Cleanup] setup: creating BigQueryWrapper') ++ self._bq_wrapper = bigquery_tools.BigQueryWrapper() ++ ++ def process( ++ self, ++ element: Tuple[str, Tuple[int, int]], ++ streams_read=beam.DoFn.StateParam(STREAMS_READ) ++ ) -> None: ++ table_key = element[0] ++ split_count = element[1][0] ++ total_streams = element[1][1] ++ ++ _LOGGER.info( ++ '[Cleanup] Received cleanup signal for %s: ' ++ 'split_count=%d, total_streams=%d', ++ table_key, ++ split_count, ++ total_streams) ++ ++ streams_read.add(split_count) ++ current_read = streams_read.read() ++ ++ _LOGGER.info( ++ '[Cleanup] State for %s: streams_read=%d/%d', ++ table_key, ++ current_read, ++ total_streams) ++ ++ if current_read >= total_streams: ++ parts = table_key.split('.') ++ if len(parts) == 3: ++ project, dataset, table = parts ++ _LOGGER.info( ++ '[Cleanup] All streams read: DELETING temp table %s', table_key) ++ self._bq_wrapper._delete_table(project, dataset, table) ++ _LOGGER.info('[Cleanup] Deleted temp table %s', table_key) ++ Metrics.counter('BigQueryChangeHistory', 'temp_tables_deleted').inc() ++ streams_read.clear() ++ else: ++ _LOGGER.info( ++ '[Cleanup] Not yet complete for %s (%d/%d), ' ++ 'waiting for more signals', ++ table_key, ++ current_read, ++ total_streams) ++ ++ ++# ============================================================================= ++# Public API: ReadBigQueryChangeHistory ++# ============================================================================= ++ ++ ++class ReadBigQueryChangeHistory(beam.PTransform): ++ """Streaming source for BigQuery change history. ++ ++ Continuously polls BigQuery APPENDS() or CHANGES() functions and emits ++ changed rows as an unbounded PCollection of dicts. ++ ++ Args: ++ table: BigQuery table to read changes from. ++ Format: 'project:dataset.table' or 'project.dataset.table'. ++ poll_interval_sec: Seconds between polls. Default 60. ++ start_time: Start reading from this timestamp (float, epoch seconds). ++ Default: current time when pipeline starts. ++ stop_time: Stop polling at this timestamp. Default: run forever. ++ change_function: 'CHANGES' or 'APPENDS'. Default 'APPENDS'. ++ buffer_sec: Safety buffer in seconds behind now(). Default 10. BQ does not ++ fail or wait if the query end_ts is less than BQ's CURRENT_TIMESTAMP. ++ This is an extra guardrail to protect against silent data. ++ project: GCP project ID. Default: from pipeline options. ++ temp_dataset: Dataset for temp tables. If None (default), a ++ per-pipeline dataset is auto-created with a 24-hour table ++ expiration as a safety net for orphaned tables. Set this to ++ use an existing dataset (e.g. if your service account lacks ++ bigquery.datasets.create permission). ++ location: BigQuery geographic location for query jobs and temp ++ dataset (e.g. 'US', 'us-central1'). If None (default), inferred ++ from the source table. ++ change_type_column: Output column name for the _CHANGE_TYPE ++ pseudo-column. Default 'change_type'. Change this if your source ++ table already has a column named 'change_type'. ++ change_timestamp_column: Output column name for the ++ _CHANGE_TIMESTAMP pseudo-column. Default 'change_timestamp'. ++ Change this if your source table already has a column named ++ 'change_timestamp'. This column is also used internally to ++ extract event timestamps for watermark tracking. ++ columns: Optional list of column names to select from the source ++ table. If None (default), all columns are selected. The ++ pseudo-columns (change_type, change_timestamp) are always ++ included regardless of this setting. ++ row_filter: Optional SQL boolean expression used as a WHERE clause ++ on the CHANGES/APPENDS query. Do not include the WHERE keyword. ++ Example: ``'status = "active" AND region = "US"'``. ++ batch_arrow_read: If True (default), convert Arrow RecordBatches in ++ bulk using to_pylist() instead of per-cell .as_py() calls. ++ This is 1.5x faster for large tables at the cost of ~2x peak ++ memory per RecordBatch. Set to False for minimal memory usage. ++ max_split_rounds: Maximum number of recursive SplitReadStream ++ rounds. Each round splits every stream at fraction=0.5, ++ potentially doubling the stream count (if BQ allows). Default ++ 1 (one round of splitting). Set 0 to disable splitting ++ entirely. Set higher for very large tables where more ++ parallelism is needed. ++ decompress_shards: If set to a positive integer, the Read SDF ++ emits raw compressed Arrow batches instead of decoded rows. ++ The batches are reshuffled for fan-out and then decoded in a ++ separate DoFn. This spreads decompression and Arrow-to-Python ++ conversion CPU across more workers. If None (default), rows ++ are decoded inline within the Read SDF. ++ """ ++ def __init__( ++ self, ++ table: str, ++ poll_interval_sec: float = 60, ++ start_time: Optional[float] = None, ++ stop_time: Optional[float] = None, ++ change_function: str = 'APPENDS', ++ buffer_sec: float = 10, ++ project: Optional[str] = None, ++ temp_dataset: Optional[str] = None, ++ location: Optional[str] = None, ++ change_type_column: str = 'change_type', ++ change_timestamp_column: str = 'change_timestamp', ++ columns: Optional[List[str]] = None, ++ row_filter: Optional[str] = None, ++ batch_arrow_read: bool = True, ++ max_split_rounds: int = 1, ++ decompress_shards: Optional[int] = None) -> None: ++ super().__init__() ++ if bq_storage is None: ++ raise ImportError( ++ 'google-cloud-bigquery-storage is required for ' ++ 'ReadBigQueryChangeHistory. Install it with: ' ++ 'pip install google-cloud-bigquery-storage') ++ if pyarrow is None: ++ raise ImportError( ++ 'pyarrow is required for ReadBigQueryChangeHistory. ' ++ 'Install it with: pip install pyarrow') ++ if change_function not in ('CHANGES', 'APPENDS'): ++ raise ValueError( ++ f"change_function must be 'CHANGES' or 'APPENDS', " ++ f"got '{change_function}'") ++ if poll_interval_sec < 15: ++ raise ValueError( ++ f'poll_interval_sec must be >= 15, got {poll_interval_sec}') ++ if buffer_sec < 0: ++ raise ValueError(f'buffer_sec must be >= 0, got {buffer_sec}') ++ self._table = table ++ self._poll_interval_sec = poll_interval_sec ++ self._start_time = start_time ++ self._stop_time = stop_time ++ self._change_function = change_function ++ self._buffer_sec = buffer_sec ++ self._project = project ++ self._temp_dataset = temp_dataset ++ self._location = location ++ self._change_type_column = change_type_column ++ self._change_timestamp_column = change_timestamp_column ++ self._columns = columns ++ self._row_filter = row_filter ++ self._batch_arrow_read = batch_arrow_read ++ self._max_split_rounds = max_split_rounds ++ self._decompress_shards = decompress_shards ++ ++ def expand(self, pbegin: beam.pvalue.PBegin) -> beam.PCollection: ++ project = self._project ++ if project is None: ++ project = pbegin.pipeline.options.view_as( ++ beam.options.pipeline_options.GoogleCloudOptions).project ++ ++ if project is None: ++ raise ValueError( ++ 'project must be specified either in ReadBigQueryChangeHistory ' ++ 'or in pipeline options (--project)') ++ ++ start_time = Timestamp(self._start_time or time.time()) ++ stop_time = ( ++ Timestamp(self._stop_time) ++ if self._stop_time is not None else MAX_TIMESTAMP) ++ buffer = Duration(seconds=self._buffer_sec) ++ poll_interval = Duration(seconds=self._poll_interval_sec) ++ ++ temp_dataset = self._temp_dataset ++ if temp_dataset is None: ++ temp_dataset = f'beam_ch_temp_{uuid.uuid4().hex[:12]}' ++ ++ _LOGGER.info( ++ '[ReadBigQueryChangeHistory] expand: table=%s, project=%s, ' ++ 'change_function=%s, poll_interval=%d sec, buffer=%d sec, ' ++ 'temp_dataset=%s, start_time=%s, stop_time=%s', ++ self._table, ++ project, ++ self._change_function, ++ self._poll_interval_sec, ++ self._buffer_sec, ++ temp_dataset, ++ _utc(start_time), ++ _utc(stop_time) if stop_time != MAX_TIMESTAMP else 'INF') ++ ++ # Custom polling SDF emits lightweight _QueryRange instructions. ++ # The SDF uses defer_remainder() for poll timing and ++ # _PollWatermarkEstimator to hold the watermark at data timestamps. ++ # On the first invocation it handles the full historical range ++ # [start_time, now - buffer) in a single poll. ++ config = _PollConfig(start_time=start_time) ++ ++ query_ranges = ( ++ pbegin ++ | 'CreatePollConfig' >> beam.Create([config]) ++ | 'PollChangeHistory' >> beam.ParDo( ++ _PollChangeHistoryFn( ++ table=self._table, ++ project=project, ++ change_function=self._change_function, ++ buffer=buffer, ++ start_time=start_time, ++ stop_time=stop_time, ++ poll_interval=poll_interval, ++ location=self._location))) ++ ++ # CommitQueryResults: Reshuffle commits _QueryResult (temp table ref) ++ # so that if the Read SDF retries, it re-reads the existing temp table ++ # instead of re-running the BQ query. ++ # Possible edge-case is that if ReadStorageStreams doesn't read the temp ++ # table within 24 hours (table expiration) it can end up in a bad state by ++ # trying to query a non-existing table. ++ query_results = ( ++ query_ranges ++ | 'CommitQueryRanges' >> beam.Reshuffle() ++ | 'ExecuteQueries' >> beam.ParDo( ++ _ExecuteQueryFn( ++ table=self._table, ++ project=project, ++ change_function=self._change_function, ++ temp_dataset=temp_dataset, ++ location=self._location, ++ change_type_column=self._change_type_column, ++ change_timestamp_column=self._change_timestamp_column, ++ columns=self._columns, ++ row_filter=self._row_filter)) ++ | 'CommitQueryResults' >> beam.Reshuffle()) ++ ++ emit_raw = self._decompress_shards is not None ++ ++ read_sdf = beam.ParDo( ++ _ReadStorageStreamsSDF( ++ batch_arrow_read=self._batch_arrow_read, ++ change_timestamp_column=self._change_timestamp_column, ++ max_split_rounds=self._max_split_rounds, ++ emit_raw_batches=emit_raw)) ++ if emit_raw: ++ read_sdf = read_sdf.with_output_types(Tuple[bytes, bytes]) ++ else: ++ read_sdf = read_sdf.with_output_types(Dict[str, Any]) ++ ++ read_outputs = ( ++ query_results ++ | 'ReadStorageStreams' >> read_sdf.with_outputs( ++ _CLEANUP_TAG, main='rows')) ++ ++ _ = ( ++ read_outputs[_CLEANUP_TAG] ++ | 'CleanupTempTables' >> beam.ParDo(_CleanupTempTablesFn())) ++ ++ if emit_raw: ++ # Fan out raw Arrow batches across decompress_shards workers ++ # via GBK, then decompress and convert to timestamped row dicts. ++ # Uses a discarding trigger so GBK fires per-element without ++ # waiting for the GlobalWindow to close. ++ num_shards = self._decompress_shards ++ rows = ( ++ read_outputs['rows'] ++ | 'ShardBatches' >> beam.WithKeys( ++ lambda _, n=num_shards: random.randint(0, n - 1)) ++ | 'WindowForGBK' >> beam.WindowInto( ++ GlobalWindows(), ++ trigger=beam_trigger.Repeatedly( ++ beam_trigger.AfterCount(1)), ++ accumulation_mode=( ++ beam_trigger.AccumulationMode.DISCARDING)) ++ | 'GroupByShardKey' >> beam.GroupByKey() ++ | 'DecompressBatches' >> beam.ParDo( ++ _DecompressArrowBatchesFn( ++ change_timestamp_column=( ++ self._change_timestamp_column)))) ++ return rows ++ else: ++ return read_outputs['rows'] +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py +new file mode 100644 +index 000000000..264da98ac +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py +@@ -0,0 +1,632 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Configurable metric computation for anomaly detection pipelines. ++ ++This module provides a ``MetricSpec`` configuration system and a ++``ComputeMetric`` PTransform that computes windowed, grouped metrics from ++raw row dicts (e.g., from ``ReadBigQueryChangeHistory``). The output is ++suitable for feeding directly into ``AnomalyDetection``. ++ ++Example usage:: ++ ++ from apache_beam.ml.anomaly.metric import ( ++ MetricSpec, AggregationSpec, WindowSpec, MeasureSpec, ++ DerivedField, WindowType, AggOp, ComputeMetric) ++ from bqmonitor.safe_eval import Expr ++ from apache_beam.ml.anomaly.transforms import AnomalyDetection ++ from apache_beam.ml.anomaly.detectors.zscore import ZScore ++ ++ # CUJ 1: Total revenue per hour ++ spec = MetricSpec( ++ name='revenue', ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=3600), ++ measures=[MeasureSpec( ++ field='transaction_amount', agg=AggOp.SUM, alias='revenue')], ++ ), ++ ) ++ result = cdc_rows | ComputeMetric(spec) | AnomalyDetection(ZScore()) ++ ++ # CUJ 2: CTR grouped by dimensions ++ spec = MetricSpec( ++ name='ctr', ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=86400), ++ group_by=['campaign_type', 'user_segment'], ++ measures=[ ++ MeasureSpec(field='is_click', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec(field='is_click', agg=AggOp.COUNT, ++ alias='impressions'), ++ ], ++ ), ++ measure_combiner=Expr.from_string("clicks / impressions"), ++ ) ++ ++ # CUJ 3: Success rate with derived field ++ spec = MetricSpec( ++ name='success_rate', ++ derived_fields=[ ++ DerivedField( ++ name='is_success', ++ expression=Expr.from_string( ++ "1 if status == 'success' else 0")), ++ ], ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=86400), ++ group_by=['brand_name', 'category'], ++ measures=[ ++ MeasureSpec(field='is_success', agg=AggOp.SUM, ++ alias='successes'), ++ MeasureSpec(field='is_success', agg=AggOp.COUNT, alias='total'), ++ ], ++ ), ++ measure_combiner=Expr.from_string("successes / total"), ++ ) ++""" ++ ++import dataclasses ++import random ++from enum import Enum ++from typing import Any ++from typing import Optional ++from typing import Tuple ++ ++import apache_beam as beam ++from apache_beam.transforms import combiners ++from apache_beam.transforms import window as beam_window ++ ++from bqmonitor.safe_eval import Expr ++from apache_beam.ml.anomaly.specifiable import specifiable ++ ++ ++class WindowType(Enum): ++ """Window type for metric aggregation.""" ++ FIXED = 'fixed' ++ SLIDING = 'sliding' ++ ++ ++class AggOp(Enum): ++ """Aggregation operator.""" ++ SUM = 'SUM' ++ COUNT = 'COUNT' ++ MIN = 'MIN' ++ MAX = 'MAX' ++ MEAN = 'MEAN' ++ ++ ++class FanoutStrategy(Enum): ++ """Strategy for global (non-keyed) aggregation parallelism. ++ ++ NONE: Plain CombineGlobally, no fanout. Relies on combiner lifting (PGBK) ++ for mapper-side pre-combining. Works well when upstream provides enough ++ parallel bundles (e.g. decompress_shards) and streaming state I/O on a ++ single key is not a bottleneck. ++ HOTKEY_FANOUT: Uses CombineGlobally.with_fanout(). Per-bundle nonce sharding. ++ Better PGBK table efficiency (1 slot per bundle), but shard distribution ++ depends on bundle count and sizes. Good for multi-key CombinePerKey where ++ only a few keys are hot. ++ SHARDED: Per-element random sharding with _PreCombineFn/_PostCombineFn. ++ Uniform distribution regardless of bundle count. One extra GBK vs NONE, ++ but distributes streaming state I/O across shard keys. Best for single ++ global key at high throughput. ++ """ ++ NONE = 'none' ++ HOTKEY_FANOUT = 'hotkey_fanout' ++ SHARDED = 'sharded' ++ ++ ++@dataclasses.dataclass(frozen=True) ++class WindowSpec: ++ """Window configuration for metric aggregation. ++ ++ Args: ++ type: FIXED or SLIDING window. ++ size_seconds: Window size in seconds. ++ period_seconds: Slide period in seconds (required for SLIDING, ignored for ++ FIXED). ++ """ ++ type: WindowType = WindowType.FIXED ++ size_seconds: int = 3600 ++ period_seconds: Optional[int] = None ++ ++ ++@dataclasses.dataclass(frozen=True) ++class DerivedField: ++ """Pre-aggregation column derivation via expression. ++ ++ Args: ++ name: Name of the new field to create. ++ expression: A compiled ``Expr`` callable, e.g. ++ ``Expr.from_string("1 if status == 'success' else 0")``. ++ """ ++ name: str ++ expression: Expr ++ ++ ++@dataclasses.dataclass(frozen=True) ++class MeasureSpec: ++ """A single aggregation measure. ++ ++ Args: ++ field: Input field name to aggregate. ++ agg: The aggregation operator. ++ alias: Output name for this measure's result. ++ """ ++ field: str ++ agg: AggOp ++ alias: str ++ ++ ++@dataclasses.dataclass(frozen=True) ++class AggregationSpec: ++ """Windowed grouped aggregation configuration. ++ ++ Args: ++ window: Window configuration. ++ group_by: Field names for grouping. Empty list means global aggregation. ++ measures: List of aggregation measures. ++ """ ++ window: WindowSpec = dataclasses.field(default_factory=WindowSpec) ++ group_by: list = dataclasses.field(default_factory=list) ++ measures: list = dataclasses.field(default_factory=list) ++ ++ ++@specifiable ++class MetricSpec: ++ """Complete metric computation specification. ++ ++ Defines how to transform raw row dicts into a single numeric metric value ++ suitable for anomaly detection. ++ ++ Args: ++ aggregation: Windowed grouped aggregation spec. ++ derived_fields: Optional pre-aggregation derived fields. ++ measure_combiner: Optional post-aggregation ``Expr`` operating on measure ++ aliases. Required when there are multiple measures. ++ name: Optional human-readable metric name. ++ """ ++ def __init__( ++ self, ++ aggregation, ++ derived_fields=None, ++ measure_combiner=None, ++ name=None, ++ ): ++ self.name = name ++ self.aggregation = aggregation ++ self.derived_fields = derived_fields or [] ++ self.measure_combiner = measure_combiner ++ self._validate() ++ ++ def _validate(self): ++ agg = self.aggregation ++ if not agg.measures: ++ raise ValueError("MetricSpec requires at least one measure") ++ if self.measure_combiner is None and len(agg.measures) > 1: ++ raise ValueError( ++ "measure_combiner is required when there are multiple measures. " ++ f"Got {len(agg.measures)} measures: " ++ f"{[m.alias for m in agg.measures]}") ++ if (agg.window.type == WindowType.SLIDING and ++ agg.window.period_seconds is None): ++ raise ValueError("period_seconds is required for SLIDING windows") ++ for df in self.derived_fields: ++ if not isinstance(df.expression, Expr): ++ raise TypeError( ++ f"DerivedField.expression must be an Expr, " ++ f"got {type(df.expression).__name__}") ++ if (self.measure_combiner is not None and ++ not isinstance(self.measure_combiner, Expr)): ++ raise TypeError( ++ f"measure_combiner must be an Expr, " ++ f"got {type(self.measure_combiner).__name__}") ++ # Validate that measure_combiner only references known measure aliases. ++ if self.measure_combiner is not None: ++ aliases = {m.alias for m in agg.measures} ++ unknown = self.measure_combiner.field_refs() - aliases ++ if unknown: ++ raise ValueError( ++ f"measure_combiner references unknown fields: {unknown}. " ++ f"Available measure aliases: {aliases}") ++ ++ def required_source_columns(self): ++ """Return the set of source table columns needed by this metric spec. ++ ++ This includes group_by fields, measure fields (excluding derived field ++ names), and field references from derived field expressions. ++ """ ++ derived_names = {df.name for df in self.derived_fields} ++ cols = set() ++ cols.update(self.aggregation.group_by) ++ for m in self.aggregation.measures: ++ if m.agg != AggOp.COUNT and m.field not in derived_names: ++ cols.add(m.field) ++ for df in self.derived_fields: ++ cols.update(df.expression.field_refs()) ++ return cols ++ ++ def to_dict(self): ++ """Serialize to a plain dict suitable for JSON.""" ++ result = { ++ 'aggregation': { ++ 'window': { ++ 'type': self.aggregation.window.type.value, ++ 'size_seconds': self.aggregation.window.size_seconds, ++ 'period_seconds': self.aggregation.window.period_seconds, ++ }, ++ 'group_by': list(self.aggregation.group_by), ++ 'measures': [{ ++ 'field': m.field, 'agg': m.agg.value, 'alias': m.alias ++ } for m in self.aggregation.measures], ++ }, ++ } ++ if self.derived_fields: ++ result['derived_fields'] = [{ ++ 'name': df.name, 'expression': str(df.expression) ++ } for df in self.derived_fields] ++ if self.measure_combiner is not None: ++ result['measure_combiner'] = {'expression': str(self.measure_combiner)} ++ if self.name is not None: ++ result['name'] = self.name ++ return result ++ ++ @classmethod ++ def from_dict(cls, d): ++ """Construct a MetricSpec from a plain dict (e.g., loaded from JSON). ++ ++ Expressions (``measure_combiner`` and ``derived_fields[].expression``) ++ are Python expression strings, e.g.:: ++ ++ "measure_combiner": {"expression": "clicks / impressions"} ++ "expression": "1 if status == 'success' else 0" ++ ++ Args: ++ d: Dictionary with keys matching the MetricSpec constructor. ++ ++ Returns: ++ MetricSpec instance. ++ ++ Raises: ++ TypeError: If an expression is not a string. ++ SyntaxError: If an expression string is not valid Python syntax. ++ ValueError: If an expression uses unsupported constructs, or if ++ measure_combiner references fields not in the measure aliases. ++ """ ++ agg_dict = d['aggregation'] ++ window_dict = agg_dict.get('window', {}) ++ window = WindowSpec( ++ type=WindowType(window_dict.get('type', 'fixed')), ++ size_seconds=window_dict.get('size_seconds', 3600), ++ period_seconds=window_dict.get('period_seconds'), ++ ) ++ measures = [ ++ MeasureSpec(field=m['field'], agg=AggOp(m['agg']), alias=m['alias']) ++ for m in agg_dict.get('measures', []) ++ ] ++ derived_fields = None ++ if 'derived_fields' in d and d['derived_fields']: ++ derived_fields = [] ++ for df in d['derived_fields']: ++ expr_val = df['expression'] ++ if not isinstance(expr_val, str): ++ raise TypeError( ++ f"derived_fields[].expression must be a string, " ++ f"got {type(expr_val).__name__}. " ++ f"Example: \"1 if status == 'success' else 0\"") ++ derived_fields.append( ++ DerivedField( ++ name=df['name'], expression=Expr.from_string(expr_val))) ++ measure_combiner = None ++ if 'measure_combiner' in d and d['measure_combiner'] is not None: ++ mc = d['measure_combiner'] ++ expr_val = mc['expression'] if isinstance(mc, dict) else mc ++ if not isinstance(expr_val, str): ++ raise TypeError( ++ f"measure_combiner.expression must be a string, " ++ f"got {type(expr_val).__name__}. " ++ f"Example: \"clicks / impressions\"") ++ measure_combiner = Expr.from_string(expr_val) ++ return cls( ++ aggregation=AggregationSpec( ++ window=window, ++ group_by=agg_dict.get('group_by', []), ++ measures=measures, ++ ), ++ derived_fields=derived_fields, ++ measure_combiner=measure_combiner, ++ name=d.get('name'), ++ _run_init=True, ++ ) ++ ++ ++# --------------------------------------------------------------------------- ++# Internal CombineFn and DoFns ++# --------------------------------------------------------------------------- ++ ++ ++class _SumCombineFn(beam.CombineFn): ++ def create_accumulator(self): ++ return 0 ++ ++ def add_input(self, accumulator, element): ++ return accumulator + element ++ ++ def merge_accumulators(self, accumulators): ++ return sum(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator ++ ++ ++class _MinCombineFn(beam.CombineFn): ++ def create_accumulator(self): ++ return float('inf') ++ ++ def add_input(self, accumulator, element): ++ return element if element < accumulator else accumulator ++ ++ def merge_accumulators(self, accumulators): ++ return min(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator ++ ++ ++class _MaxCombineFn(beam.CombineFn): ++ def create_accumulator(self): ++ return float('-inf') ++ ++ def add_input(self, accumulator, element): ++ return element if element > accumulator else accumulator ++ ++ def merge_accumulators(self, accumulators): ++ return max(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator ++ ++ ++def _get_combiner_for_agg(agg_op): ++ """Map AggOp enum to a Beam CombineFn instance.""" ++ if agg_op == AggOp.SUM: ++ return _SumCombineFn() ++ elif agg_op == AggOp.COUNT: ++ return combiners.CountCombineFn() ++ elif agg_op == AggOp.MIN: ++ return _MinCombineFn() ++ elif agg_op == AggOp.MAX: ++ return _MaxCombineFn() ++ elif agg_op == AggOp.MEAN: ++ return combiners.MeanCombineFn() ++ else: ++ raise ValueError(f"Unknown aggregation operator: {agg_op}") ++ ++ ++class _PreCombineFn(beam.CombineFn): ++ """Stage 1 wrapper: extract_output returns the raw accumulator.""" ++ def __init__(self, combine_fn): ++ self._combine_fn = combine_fn ++ ++ def create_accumulator(self): ++ return self._combine_fn.create_accumulator() ++ ++ def add_input(self, accumulator, element): ++ return self._combine_fn.add_input(accumulator, element) ++ ++ def merge_accumulators(self, accumulators): ++ return self._combine_fn.merge_accumulators(accumulators) ++ ++ def extract_output(self, accumulator): ++ return accumulator # pass raw accumulator, NOT final output ++ ++ ++class _PostCombineFn(beam.CombineFn): ++ """Stage 2 wrapper: add_input merges an accumulator from Stage 1.""" ++ def __init__(self, combine_fn): ++ self._combine_fn = combine_fn ++ ++ def create_accumulator(self): ++ return self._combine_fn.create_accumulator() ++ ++ def add_input(self, accumulator, element): ++ return self._combine_fn.merge_accumulators([accumulator, element]) ++ ++ def merge_accumulators(self, accumulators): ++ return self._combine_fn.merge_accumulators(accumulators) ++ ++ def extract_output(self, accumulator): ++ return self._combine_fn.extract_output(accumulator) ++ ++ ++class _DerivedFieldsFn: ++ """Callable that evaluates derived field expressions on each row dict. ++ ++ Each derived field's ``expression`` is a compiled ``Expr`` callable. ++ This class is passed to ``beam.Map`` and is pickle-safe because ``Expr`` ++ implements ``__reduce__``. ++ """ ++ def __init__(self, derived_fields): ++ self._fields = [(df.name, df.expression) for df in derived_fields] ++ ++ def __call__(self, row): ++ row = dict(row) ++ for name, expr in self._fields: ++ row[name] = expr(row) ++ return row ++ ++ ++class _ApplyMetricExpr(beam.DoFn): ++ """DoFn that evaluates a post-aggregation expression on combined results.""" ++ def __init__(self, measure_combiner, is_keyed): ++ self._measure_combiner = measure_combiner ++ self._is_keyed = is_keyed ++ ++ def process(self, element, window=beam.DoFn.WindowParam): ++ if self._is_keyed: ++ key, agg_dict = element ++ else: ++ agg_dict = element ++ ++ if self._measure_combiner is not None: ++ value = float(self._measure_combiner(agg_dict)) ++ else: ++ value = float(next(iter(agg_dict.values()))) ++ ++ row = beam.Row( ++ value=value, ++ window_start=float(window.start), ++ window_end=float(window.end)) ++ ++ if self._is_keyed: ++ yield (key, row) ++ else: ++ yield row ++ ++ ++class ComputeMetric(beam.PTransform): ++ """Transforms raw row dicts into metric beam.Rows for anomaly detection. ++ ++ Takes a ``PCollection[dict]`` with event-time timestamps and produces ++ either ``PCollection[beam.Row]`` (for global aggregation) or ++ ``PCollection[tuple[key, beam.Row]]`` (for grouped aggregation). ++ ++ The output is directly compatible with ``AnomalyDetection``. ++ ++ Args: ++ metric_spec: A ``MetricSpec`` defining the metric computation. ++ fanout_strategy: Strategy for global (non-keyed) aggregation parallelism. ++ Ignored when group_by is set. Default: SHARDED. ++ fanout: Number of shards for SHARDED or HOTKEY_FANOUT strategies. ++ Ignored for NONE. Default: 400. ++ """ ++ def __init__(self, metric_spec, fanout_strategy=FanoutStrategy.SHARDED, ++ fanout=400): ++ super().__init__() ++ self._spec = metric_spec ++ self._fanout_strategy = fanout_strategy ++ self._fanout = fanout ++ ++ def expand(self, pcoll): ++ spec = self._spec ++ agg = spec.aggregation ++ ++ # Step 1: Apply derived fields ++ if spec.derived_fields: ++ pcoll = pcoll | 'DerivedFields' >> beam.Map( ++ _DerivedFieldsFn(spec.derived_fields)) ++ ++ # Step 2: Apply windowing ++ if agg.window.type == WindowType.FIXED: ++ window_fn = beam_window.FixedWindows(agg.window.size_seconds) ++ elif agg.window.type == WindowType.SLIDING: ++ window_fn = beam_window.SlidingWindows( ++ agg.window.size_seconds, agg.window.period_seconds) ++ else: ++ raise ValueError(f"Unknown window type: {agg.window.type}") ++ ++ windowed = pcoll | 'Window' >> beam.WindowInto(window_fn) ++ ++ # Step 3: Aggregate ++ measures = agg.measures ++ aliases = [m.alias for m in measures] ++ is_keyed = bool(agg.group_by) ++ ++ # Single-measure optimization: skip TupleCombineFn overhead (avoids ++ # tuple creation/unpacking per element on the hot path). ++ if len(measures) == 1: ++ combine_fn = _get_combiner_for_agg(measures[0].agg) ++ _m0 = measures[0] ++ _a0 = aliases[0] ++ ++ def extract_fields(row_dict): ++ return row_dict.get(_m0.field) if _m0.agg != AggOp.COUNT else 1 ++ ++ def to_alias_dict(value): ++ return {_a0: value} ++ else: ++ combine_fn = combiners.TupleCombineFn( ++ *[_get_combiner_for_agg(m.agg) for m in measures]) ++ ++ def extract_fields(row_dict): ++ return tuple( ++ row_dict.get(m.field) if m.agg != AggOp.COUNT else 1 ++ for m in measures) ++ ++ def to_alias_dict(values): ++ return dict(zip(aliases, values)) ++ ++ if is_keyed: ++ group_by_fields = agg.group_by ++ ++ def extract_key_and_fields(row_dict): ++ key = tuple(row_dict.get(f) for f in group_by_fields) ++ return (key, extract_fields(row_dict)) ++ ++ keyed = windowed | 'ExtractKey' >> beam.Map(extract_key_and_fields) ++ aggregated = ( ++ keyed ++ | 'Combine' >> beam.CombinePerKey(combine_fn) ++ | 'ToDict' >> beam.MapTuple(lambda k, v: (k, to_alias_dict(v)))) ++ else: ++ strategy = self._fanout_strategy ++ if strategy == FanoutStrategy.NONE: ++ aggregated = ( ++ windowed ++ | 'ExtractFields' >> beam.Map(extract_fields) ++ | 'Combine' >> beam.CombineGlobally( ++ combine_fn).without_defaults() ++ | 'ToDict' >> beam.Map(to_alias_dict)) ++ elif strategy == FanoutStrategy.HOTKEY_FANOUT: ++ aggregated = ( ++ windowed ++ | 'ExtractFields' >> beam.Map(extract_fields) ++ | 'Combine' >> beam.CombineGlobally( ++ combine_fn).with_fanout(self._fanout).without_defaults() ++ | 'ToDict' >> beam.Map(to_alias_dict)) ++ elif strategy == FanoutStrategy.SHARDED: ++ _num_shards = self._fanout ++ ++ def _shard_fields(row_dict): ++ return (random.randint(0, _num_shards - 1), ++ extract_fields(row_dict)) ++ ++ pre_fn = _PreCombineFn(combine_fn) ++ post_fn = _PostCombineFn(combine_fn) ++ aggregated = ( ++ windowed ++ | 'ShardAndExtract' >> beam.Map(_shard_fields) ++ | 'PartialCombine' >> beam.CombinePerKey(pre_fn) ++ | 'DropShard' >> beam.Values() ++ | 'FinalCombine' >> beam.CombineGlobally( ++ post_fn).without_defaults() ++ | 'ToDict' >> beam.Map(to_alias_dict)) ++ else: ++ raise ValueError(f"Unknown fanout strategy: {strategy}") ++ ++ # Step 4: Apply metric expression and set output type hints ++ metric_dofn = _ApplyMetricExpr(spec.measure_combiner, is_keyed) ++ ++ if is_keyed: ++ # AnomalyDetection checks isinstance(element_type, TupleConstraint) ++ # to detect keyed input. We must annotate the output type. ++ result = aggregated | 'MetricExpr' >> beam.ParDo( ++ metric_dofn).with_output_types(Tuple[Any, beam.Row]) ++ else: ++ result = aggregated | 'MetricExpr' >> beam.ParDo( ++ metric_dofn).with_output_types(beam.Row) ++ ++ return result +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py +new file mode 100644 +index 000000000..9e6102816 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py +@@ -0,0 +1,1053 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Anomaly monitoring pipeline for BigQuery tables. ++ ++Reads streaming CDC data from BigQuery, computes a configurable windowed ++metric, runs anomaly detection, and publishes anomalies to Pub/Sub. ++ ++Designed to be run as a Dataflow Flex Template or locally with DirectRunner. ++ ++Usage (Flex Template):: ++ ++ gcloud dataflow flex-template run "sales-monitor-$(date +%Y%m%d)" \\ ++ --template-file-gcs-location "gs://bucket/anomaly_monitor.json" \\ ++ --parameters table="project:dataset.table" \\ ++ --parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' \\ ++ --parameters detector_spec='{"type":"ZScore"}' \\ ++ --region us-central1 ++ ++Usage (PrismRunner):: ++ ++ python main.py \\ ++ --table=project:dataset.table \\ ++ --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' \\ ++ --detector_spec='{"type":"ZScore"}' \\ ++ --runner=PrismRunner ++ ++Usage (DataflowRunner):: ++ ++ python main.py \\ ++ --table=project:dataset.table \\ ++ --metric_spec='' \\ ++ --detector_spec='' \\ ++ --runner=DataflowRunner \\ ++ --project=my-project \\ ++ --region=us-central1 \\ ++ --temp_location=gs://bucket/temp \\ ++ --staging_location=gs://bucket/staging \\ ++ --setup_file=./setup.py ++ ++ ++metric_spec JSON Reference ++========================== ++ ++Top-level ``metric_spec`` object:: ++ ++ { ++ "aggregation": { ... }, # required ++ "derived_fields": [ ... ], # optional, pre-aggregation ++ "measure_combiner": { ... } # optional (required if >1 measure) ++ } ++ ++aggregation ++----------- ++:: ++ ++ "aggregation": { ++ "window": { ++ "type": "fixed" | "sliding", ++ "size_seconds": , # window size in seconds ++ "period_seconds": # slide period (required for sliding) ++ }, ++ "group_by": ["field1", "field2"], # optional, omit for global agg ++ "measures": [ ++ {"field": "", "agg": "", "alias": ""}, ++ ... ++ ] ++ } ++ ++Aggregation operators (``agg``): ``SUM``, ``COUNT``, ``MIN``, ``MAX``, ``MEAN``. ++ ++For ``COUNT``, the ``field`` value is ignored — it counts all rows in the ++group. ++ ++Expressions ++----------- ++Both ``measure_combiner.expression`` and ``derived_fields[].expression`` ++are Python expression strings. Bare names are field references, and the ++following syntax is supported: ++ ++- Arithmetic: ``+``, ``-``, ``*``, ``/``, ``//``, ``%``, ``**`` ++- Comparisons: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=`` ++- Boolean logic: ``and``, ``or``, ``not`` ++- Negation: ``-field`` ++- Conditional: ``true_val if condition else false_val`` ++- Functions: ``abs()``, ``min()``, ``max()``, ``round()`` ++- Grouping: parentheses for precedence ++ ++``measure_combiner`` references measure aliases and is validated at ++pipeline construction time. ++ ++derived_fields ++-------------- ++Computed before aggregation. Each entry creates a new column available to ++measures:: ++ ++ "derived_fields": [ ++ {"name": "is_success", "expression": "1 if status == 'success' else 0"} ++ ] ++ ++measure_combiner ++---------------- ++Post-aggregation expression that combines measure aliases into a single ++value. Required when there are multiple measures (e.g., ratio metrics):: ++ ++ "measure_combiner": {"expression": "clicks / impressions"} ++ "measure_combiner": {"expression": "(successes + partial) / total"} ++ ++ ++detector_spec JSON Reference ++============================= ++ ++Top-level ``detector_spec`` object:: ++ ++ {"type": "", "config": { ... }} ++ ++The ``type`` must be a registered ``@specifiable`` detector class name. ++``config`` keys map to that class's ``__init__`` parameters plus inherited ++``AnomalyDetector`` parameters. ++ ++Common AnomalyDetector parameters (all detectors):: ++ ++ "config": { ++ "threshold_criterion": { ... }, # optional, see below ++ "model_id": "" # optional detector ID ++ } ++ ++``features`` is automatically set to ``['value']`` to match ++``ComputeMetric`` output; it does not need to be specified. ++ ++window_size ++----------- ++All detectors maintain an internal sliding window of recent values for their ++statistical trackers (mean, stdev, quantiles, etc.). The default is 1000 ++data points. Use ``window_size`` as a shorthand to override this for all ++internal trackers at once:: ++ ++ {"type": "ZScore", "config": {"window_size": 500}} ++ ++Available detectors ++------------------- ++ ++**ZScore** — ``|value - mean| / stdev`` (default threshold: 3):: ++ ++ {"type": "ZScore"} ++ ++**IQR** — Interquartile Range (default threshold: 1.5):: ++ ++ {"type": "IQR"} ++ ++**RobustZScore** — Modified Z-Score using median/MAD (default threshold: 3.5):: ++ ++ {"type": "RobustZScore"} ++ ++threshold_criterion ++------------------- ++Override the default threshold by nesting a specifiable threshold object. ++ ++**FixedThreshold** — static cutoff (scores >= cutoff are outliers):: ++ ++ "threshold_criterion": { ++ "type": "FixedThreshold", ++ "config": {"cutoff": 10} ++ } ++ ++**QuantileThreshold** — dynamic cutoff at a quantile of observed scores:: ++ ++ "threshold_criterion": { ++ "type": "QuantileThreshold", ++ "config": {"quantile": 0.95} ++ } ++ ++Both accept optional ``normal_label`` (default 0), ``outlier_label`` ++(default 1), and ``missing_label`` (default -2). ++ ++**Threshold** — fixed threshold alert based on a boolean expression. ++No warmup period, no history buffer. Alerts whenever the expression ++evaluates to true:: ++ ++ {"type": "Threshold", "expression": "value >= 0.5"} ++ {"type": "Threshold", "expression": "value > 100 or value < -100"} ++ {"type": "Threshold", "expression": "value <= 0.01"} ++ ++The expression receives the computed metric as ``value`` and supports ++all safe expression operators (see Expressions section above). ++ ++ ++Examples ++-------- ++ ++Simple SUM metric with ZScore:: ++ ++ --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' ++ --detector_spec='{"type":"ZScore"}' ++ ++Grouped ratio metric (CTR) with ZScore:: ++ ++ --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":10},"group_by":["campaign_type","browser_version"],"measures":[{"field":"is_click","agg":"SUM","alias":"clicks"},{"field":"is_click","agg":"COUNT","alias":"impressions"}]},"measure_combiner":{"expression":"clicks / impressions"}}' ++ --detector_spec='{"type":"ZScore"}' ++ ++Derived field + ratio + custom threshold:: ++ ++ --metric_spec='{"derived_fields":[{"name":"is_success","expression":"1 if status == \\'success\\' else 0"}],"aggregation":{"window":{"type":"fixed","size_seconds":10},"group_by":["brand_name","category"],"measures":[{"field":"is_success","agg":"SUM","alias":"successes"},{"field":"is_success","agg":"COUNT","alias":"total"}]},"measure_combiner":{"expression":"successes / total"}}' ++ --detector_spec='{"type":"ZScore","config":{"threshold_criterion":{"type":"FixedThreshold","config":{"cutoff":10}}}}' ++""" ++ ++import datetime ++import json ++import logging ++import re ++import time ++ ++import apache_beam as beam ++from apache_beam.io.gcp.bigquery import WriteToBigQuery ++from apache_beam.io.gcp.pubsub import WriteToPubSub ++from apache_beam.options.pipeline_options import PipelineOptions ++from apache_beam.options.pipeline_options import SetupOptions ++ ++from bqmonitor.metric import ComputeMetric ++from bqmonitor.metric import FanoutStrategy ++from bqmonitor.metric import MetricSpec ++from bqmonitor.safe_eval import Expr ++from apache_beam.ml.anomaly.base import AnomalyPrediction ++from apache_beam.ml.anomaly.base import AnomalyResult ++from apache_beam.ml.anomaly.specifiable import Spec ++from apache_beam.ml.anomaly.specifiable import Specifiable ++from apache_beam.ml.anomaly.transforms import AnomalyDetection ++ ++# Import detectors so they register with @specifiable before from_spec. ++from apache_beam.ml.anomaly.detectors import zscore # noqa: F401 ++from apache_beam.ml.anomaly.detectors import iqr # noqa: F401 ++from apache_beam.ml.anomaly.detectors import robust_zscore # noqa: F401 ++ ++_LOGGER = logging.getLogger(__name__) ++ ++_SUPPORTED_DETECTORS = ('ZScore', 'IQR', 'RobustZScore') ++ ++# Matches project:dataset.table or project.dataset.table ++_TABLE_RE = re.compile( ++ r'^[a-zA-Z0-9][a-zA-Z0-9_-]*[:\.][a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$') ++ ++ ++# --------------------------------------------------------------------------- ++# Helpers ++# --------------------------------------------------------------------------- ++ ++ ++def _unpack_result(element): ++ """Unpack a possibly-keyed AnomalyResult element. ++ ++ Returns: ++ (key, result) where key is None for unkeyed elements. ++ """ ++ if isinstance(element, tuple) and len(element) == 2: ++ return element[0], element[1] ++ return None, element ++ ++ ++def _parse_table_ref(table): ++ """Parse and validate a table reference string. ++ ++ Args: ++ table: Table reference in 'project:dataset.table' or ++ 'project.dataset.table' format. ++ ++ Returns: ++ (project, dataset, table_name) tuple. ++ ++ Raises: ++ ValueError: If the table string doesn't match the expected format. ++ """ ++ if not _TABLE_RE.match(table): ++ raise ValueError( ++ f"Invalid --table format: '{table}'. " ++ f"Expected: project:dataset.table or project.dataset.table") ++ if ':' in table: ++ project, rest = table.split(':', 1) ++ dataset, table_name = rest.split('.', 1) ++ else: ++ project, dataset, table_name = table.split('.', 2) ++ return project, dataset, table_name ++ ++ ++# --------------------------------------------------------------------------- ++# DoFns ++# --------------------------------------------------------------------------- ++ ++ ++class _LogAnomalyResult(beam.DoFn): ++ """Logs each AnomalyResult at WARNING level for visibility in Dataflow.""" ++ def process(self, element): ++ key, result = _unpack_result(element) ++ prediction = result.predictions[0] ++ example = result.example ++ ++ if prediction.label == 1: ++ tag = '!! OUTLIER !!' ++ elif prediction.label == 0: ++ tag = 'NORMAL' ++ else: ++ tag = 'WARMUP' ++ ++ ws = datetime.datetime.fromtimestamp( ++ example.window_start, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ we = datetime.datetime.fromtimestamp( ++ example.window_end, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ window_str = f'{ws}-{we}' ++ ++ if key is not None: ++ _LOGGER.warning( ++ '[%s] window=%s key=%s value=%.2f score=%s label=%s', ++ tag, window_str, key, example.value, prediction.score, ++ prediction.label) ++ else: ++ _LOGGER.warning( ++ '[%s] window=%s value=%.2f score=%s label=%s', ++ tag, window_str, example.value, prediction.score, ++ prediction.label) ++ yield element ++ ++ ++class _ThresholdAlert(beam.DoFn): ++ """Evaluates a threshold expression against metric values. ++ ++ Emits AnomalyResult elements consistent with the statistical detectors, ++ allowing threshold alerts to flow through the same logging and Pub/Sub ++ pipeline. ++ ++ The expression is evaluated with ``value`` bound to the metric value. ++ If it evaluates to truthy, the element is labelled as an outlier (1); ++ otherwise it is labelled normal (0). ++ ++ Example expressions: ``value >= 0.5``, ``value <= 0.01``, ++ ``value > 100 or value < -100``. ++ """ ++ ++ def __init__(self, expression_text): ++ self._expression_text = expression_text ++ self._expr = None ++ ++ def setup(self): ++ self._expr = Expr(self._expression_text) ++ ++ def process(self, element): ++ if isinstance(element, tuple) and len(element) == 2: ++ key, row = element ++ else: ++ key, row = None, element ++ ++ value = row.value ++ is_alert = bool(self._expr({'value': value})) ++ ++ prediction = AnomalyPrediction( ++ model_id=f'Threshold({self._expression_text})', ++ score=None, ++ label=1 if is_alert else 0, ++ threshold=None) ++ ++ result = AnomalyResult(example=row, predictions=[prediction]) ++ ++ if key is not None: ++ yield (key, result) ++ else: ++ yield result ++ ++ ++class _FormatAnomalyAsJson(beam.DoFn): ++ """Converts anomaly results (label == 1) to JSON byte strings for Pub/Sub.""" ++ def process(self, element): ++ key, result = _unpack_result(element) ++ prediction = result.predictions[0] ++ if prediction.label != 1: ++ return ++ ++ example = result.example ++ ws = datetime.datetime.fromtimestamp( ++ example.window_start, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ we = datetime.datetime.fromtimestamp( ++ example.window_end, tz=datetime.timezone.utc).strftime( ++ '%Y-%m-%dT%H:%M:%S.%fZ') ++ ++ payload = { ++ 'event_description': ( ++ f'Anomaly detected value={example.value}' ++ f' score={prediction.score}' ++ f' in window={ws}-{we}'), ++ 'agent_id': prediction.model_id, ++ } ++ if key is not None: ++ payload['key'] = str(key) ++ ++ yield json.dumps(payload).encode('utf-8') ++ ++ ++_SINK_SCHEMA = { ++ 'fields': [ ++ {'name': 'window_start', 'type': 'TIMESTAMP', 'mode': 'REQUIRED'}, ++ {'name': 'window_end', 'type': 'TIMESTAMP', 'mode': 'REQUIRED'}, ++ {'name': 'value', 'type': 'FLOAT64', 'mode': 'REQUIRED'}, ++ {'name': 'score', 'type': 'FLOAT64', 'mode': 'NULLABLE'}, ++ {'name': 'label', 'type': 'INT64', 'mode': 'REQUIRED'}, ++ {'name': 'key', 'type': 'STRING', 'mode': 'NULLABLE'}, ++ ] ++} ++ ++ ++class _FormatResultForBQ(beam.DoFn): ++ """Converts all AnomalyResult elements to BQ row dicts.""" ++ def process(self, element): ++ key, result = _unpack_result(element) ++ prediction = result.predictions[0] ++ example = result.example ++ ++ row = { ++ 'window_start': datetime.datetime.fromtimestamp( ++ example.window_start, tz=datetime.timezone.utc).isoformat(), ++ 'window_end': datetime.datetime.fromtimestamp( ++ example.window_end, tz=datetime.timezone.utc).isoformat(), ++ 'value': float(example.value), ++ 'score': float(prediction.score) if prediction.score is not None ++ else None, ++ 'label': int(prediction.label), ++ } ++ if key is not None: ++ row['key'] = str(key) ++ ++ yield row ++ ++ ++# --------------------------------------------------------------------------- ++# Pipeline options ++# --------------------------------------------------------------------------- ++ ++ ++class AnomalyMonitorOptions(PipelineOptions): ++ """Pipeline options for the anomaly monitor.""" ++ @classmethod ++ def _add_argparse_args(cls, parser): ++ parser.add_argument( ++ '--table', ++ default=None, ++ help='BigQuery table to monitor. ' ++ 'Format: project:dataset.table') ++ parser.add_argument( ++ '--metric_spec', ++ default=None, ++ help='JSON string defining the metric computation. ' ++ 'See MetricSpec.from_dict() for schema.') ++ parser.add_argument( ++ '--detector_spec', ++ default=None, ++ help='JSON string defining the anomaly detector. ' ++ 'Format: {"type":"ZScore"} or ' ++ '{"type":"ZScore","config":{"threshold_criterion":{...}}}') ++ parser.add_argument( ++ '--poll_interval_sec', ++ type=int, ++ default=60, ++ help='Seconds between BigQuery CDC polls. Default 60.') ++ parser.add_argument( ++ '--change_function', ++ default='APPENDS', ++ choices=['APPENDS', 'CHANGES'], ++ help='BigQuery change function to use. Default APPENDS.') ++ parser.add_argument( ++ '--buffer_sec', ++ type=float, ++ default=15.0, ++ help='Safety buffer behind now() in seconds. Default 15.') ++ parser.add_argument( ++ '--start_offset_sec', ++ type=float, ++ default=60.0, ++ help='Start reading from this many seconds ago. Default 60.') ++ parser.add_argument( ++ '--duration_sec', ++ type=float, ++ default=0.0, ++ help='How long to run in seconds. 0 means run forever. Default 0.') ++ parser.add_argument( ++ '--temp_dataset', ++ default=None, ++ help='BigQuery dataset for temp tables. If unset, auto-created.') ++ parser.add_argument( ++ '--topic', ++ default=None, ++ help='Pub/Sub topic name for anomaly results.') ++ parser.add_argument( ++ '--log_all_results', ++ default='false', ++ help='Log all anomaly detection results (normal, outlier, warmup) ' ++ 'at WARNING level. Default: false.') ++ parser.add_argument( ++ '--sink_table', ++ default=None, ++ help='BigQuery table to write all anomaly detection results to. ' ++ 'Format: project:dataset.table. If unset, results are not written ' ++ 'to BigQuery.') ++ parser.add_argument( ++ '--write_method', ++ default='STORAGE_WRITE_API', ++ choices=[ ++ 'STORAGE_WRITE_API', 'DEFAULT', 'FILE_LOADS', ++ 'STREAMING_INSERTS'], ++ help='BigQuery write method for the sink table. ' ++ 'Default: STORAGE_WRITE_API.') ++ parser.add_argument( ++ '--decompress_shards', ++ type=int, ++ default=400, ++ help='Number of shards for CDC Arrow batch decompression fan-out. ' ++ 'Spreads decompression CPU across workers. ' ++ '0 disables fan-out (decode inline). Default: 400.') ++ parser.add_argument( ++ '--fanout_strategy', ++ default='sharded', ++ choices=['sharded', 'hotkey_fanout', 'none'], ++ help='Parallelism strategy for global (non-keyed) metric ' ++ 'aggregation. Ignored when group_by is set. Default: sharded.') ++ parser.add_argument( ++ '--fanout', ++ type=int, ++ default=400, ++ help='Number of shards for sharded or hotkey_fanout strategies. ' ++ 'Ignored for none. Default: 400.') ++ ++ ++# --------------------------------------------------------------------------- ++# Spec parsing ++# --------------------------------------------------------------------------- ++ ++ ++def _parse_metric_spec(json_str): ++ """Parse a MetricSpec from a JSON string. ++ ++ Raises: ++ ValueError: If the JSON is malformed or the spec is invalid. ++ """ ++ try: ++ d = json.loads(json_str) ++ except json.JSONDecodeError as e: ++ raise ValueError( ++ f"Invalid JSON in --metric_spec: {e}. " ++ f"Value: {json_str[:200]}") from e ++ try: ++ return MetricSpec.from_dict(d) ++ except (ValueError, TypeError, KeyError) as e: ++ raise ValueError(f"Invalid --metric_spec: {e}") from e ++ ++ ++def _dict_to_spec(d): ++ """Recursively convert nested dicts with ``type`` keys into Spec objects. ++ ++ ``json.loads`` produces plain dicts, but ``Specifiable.from_spec`` expects ++ ``Spec`` objects for nested specifiables (e.g. ``threshold_criterion`` ++ inside a detector config). Without this conversion the nested dict passes ++ through ``_specifiable_from_spec_helper`` unchanged and the detector ++ receives a raw dict instead of the expected ``ThresholdFn`` instance. ++ """ ++ if isinstance(d, dict) and 'type' in d: ++ config = d.get('config', {}) ++ if config: ++ config = {k: _dict_to_spec(v) for k, v in config.items()} ++ return Spec(type=d['type'], config=config) ++ if isinstance(d, list): ++ return [_dict_to_spec(item) for item in d] ++ return d ++ ++ ++def _expand_window_size(d): ++ """Expand ``window_size`` shorthand into detector-specific tracker configs. ++ ++ Instead of constructing deeply nested tracker specs, users can write:: ++ ++ {"type": "ZScore", "config": {"window_size": 500}} ++ ++ This expands into the full nested tracker configuration that each detector ++ type expects. If the user already set explicit tracker configs, those take ++ precedence (``setdefault`` semantics). ++ ++ Raises: ++ ValueError: If window_size is not a positive integer. ++ """ ++ config = d.get('config', {}) ++ ws = config.pop('window_size', None) ++ if ws is None: ++ return ++ ++ if not isinstance(ws, int) or ws <= 0: ++ raise ValueError( ++ f"window_size must be a positive integer, got {ws!r}") ++ ++ detector_type = d['type'] ++ ++ if detector_type == 'ZScore': ++ config.setdefault( ++ 'sub_stat_tracker', ++ {'type': 'IncSlidingMeanTracker', 'config': {'window_size': ws}}) ++ config.setdefault( ++ 'stdev_tracker', ++ {'type': 'IncSlidingStdevTracker', 'config': {'window_size': ws}}) ++ elif detector_type == 'IQR': ++ config.setdefault( ++ 'q1_tracker', ++ { ++ 'type': 'BufferedSlidingQuantileTracker', ++ 'config': {'window_size': ws, 'q': 0.25} ++ }) ++ # q3_tracker auto-derives from q1_tracker in IQR.__init__ ++ elif detector_type == 'RobustZScore': ++ _median_tracker_spec = { ++ 'type': 'MedianTracker', ++ 'config': { ++ 'quantile_tracker': { ++ 'type': 'BufferedSlidingQuantileTracker', ++ 'config': {'window_size': ws, 'q': 0.5} ++ } ++ } ++ } ++ config.setdefault( ++ 'mad_tracker', ++ { ++ 'type': 'MadTracker', ++ 'config': { ++ 'median_tracker': _median_tracker_spec, ++ 'diff_median_tracker': { ++ 'type': 'MedianTracker', ++ 'config': { ++ 'quantile_tracker': { ++ 'type': 'BufferedSlidingQuantileTracker', ++ 'config': {'window_size': ws, 'q': 0.5} ++ } ++ } ++ } ++ } ++ }) ++ ++ ++def _parse_detector_spec(json_str): ++ """Parse an anomaly detector from a JSON Spec string. ++ ++ The JSON should have the form:: ++ ++ {"type": "ZScore"} ++ ++ Nested specifiable objects (e.g. ``threshold_criterion``) are supported:: ++ ++ {"type": "ZScore", "config": { ++ "threshold_criterion": {"type": "FixedThreshold", "config": {"cutoff": 10}} ++ }} ++ ++ A ``window_size`` shorthand sets the history buffer for all internal ++ trackers:: ++ ++ {"type": "ZScore", "config": {"window_size": 500}} ++ ++ **Threshold** — a simple fixed-threshold alerter that evaluates a boolean ++ expression against the metric value. No warmup period, no history:: ++ ++ {"type": "Threshold", "expression": "value >= 0.5"} ++ {"type": "Threshold", "expression": "value > 100 or value < -100"} ++ ++ The expression may use ``value`` (the computed metric) and all safe ++ expression operators (see Expressions section above). ++ ++ For statistical detectors, the ``type`` field must match a registered ++ @specifiable detector class (e.g. ZScore, IQR, RobustZScore). ++ ++ ``features`` is automatically set to ``['value']`` to match the output of ++ ``ComputeMetric``. Any user-supplied ``features`` is overwritten. ++ ++ Returns: ++ For statistical detectors: an instantiated AnomalyDetector. ++ For Threshold: a ``_ThresholdAlert`` DoFn instance. ++ ++ Raises: ++ ValueError: If the JSON is malformed, detector type is unknown, or ++ the spec is otherwise invalid. ++ """ ++ try: ++ d = json.loads(json_str) ++ except json.JSONDecodeError as e: ++ raise ValueError( ++ f"Invalid JSON in --detector_spec: {e}. " ++ f"Value: {json_str[:200]}") from e ++ ++ if not isinstance(d, dict) or 'type' not in d: ++ raise ValueError( ++ "detector_spec must be a JSON object with a 'type' field. " ++ f"Example: {{\"type\":\"ZScore\"}}. Got: {json_str[:200]}") ++ ++ detector_type = d['type'] ++ ++ if detector_type == 'Threshold': ++ expr_text = d.get('expression') ++ if not expr_text: ++ raise ValueError( ++ "Threshold detector requires an 'expression' field. " ++ "Example: {\"type\":\"Threshold\",\"expression\":\"value >= 0.5\"}") ++ # Validate the expression at parse time. ++ try: ++ expr = Expr(expr_text) ++ except (ValueError, SyntaxError) as e: ++ raise ValueError( ++ f"Invalid threshold expression: {e}") from e ++ if 'value' not in expr.field_refs(): ++ _LOGGER.warning( ++ "Threshold expression '%s' does not reference 'value'. " ++ "It will receive the computed metric value as 'value'.", expr_text) ++ return _ThresholdAlert(expr_text) ++ ++ if detector_type not in _SUPPORTED_DETECTORS: ++ raise ValueError( ++ f"Unknown detector type '{detector_type}'. " ++ f"Supported detectors: {', '.join(_SUPPORTED_DETECTORS)}, Threshold") ++ ++ d.setdefault('config', {}) ++ d['config']['features'] = ['value'] ++ _expand_window_size(d) ++ spec = _dict_to_spec(d) ++ try: ++ return Specifiable.from_spec(spec, _run_init=True) ++ except (ValueError, TypeError) as e: ++ raise ValueError( ++ f"Failed to construct {detector_type} detector: {e}") from e ++ ++ ++# --------------------------------------------------------------------------- ++# Preflight checks ++# --------------------------------------------------------------------------- ++ ++ ++def _preflight_checks(options): ++ """Validate GCP resources are accessible before building the pipeline. ++ ++ Checks: ++ - BigQuery source table exists and is readable. ++ - BigQuery temp dataset is writable (if specified) or datasets.create ++ permission exists (dry-run only — does not actually create). ++ - BigQuery query job dry-run succeeds (validates CDC function access). ++ - Pub/Sub topic exists. ++ ++ Logs warnings and continues if a check cannot be performed (e.g. missing ++ client library). Raises ValueError on definite failures. ++ """ ++ project, dataset, table_name = _parse_table_ref(options.table) ++ topic_path = f'projects/{project}/topics/{options.topic}' ++ ++ _check_bq_source_table(project, dataset, table_name, options) ++ _check_bq_temp_dataset(project, options) ++ _check_pubsub_topic(topic_path) ++ ++ ++def _check_bq_source_table(project, dataset, table_name, options): ++ """Verify the source BigQuery table exists and is accessible.""" ++ try: ++ from apache_beam.io.gcp import bigquery_tools ++ from apache_beam.io.gcp.internal.clients import bigquery ++ except ImportError: ++ _LOGGER.warning( ++ '[Preflight] Skipping BQ table check: ' ++ 'BigQuery client libraries not available') ++ return ++ ++ try: ++ bq = bigquery_tools.BigQueryWrapper() ++ bq.get_table(project, dataset, table_name) ++ _LOGGER.info( ++ '[Preflight] Source table %s:%s.%s is accessible', ++ project, dataset, table_name) ++ except Exception as e: ++ raise ValueError( ++ f"Cannot access BigQuery table '{project}:{dataset}.{table_name}'. " ++ f"Verify it exists and the service account has " ++ f"bigquery.tables.get and bigquery.tables.getData permissions. " ++ f"Error: {e}") from e ++ ++ # Dry-run a CDC query to verify APPENDS/CHANGES access. ++ try: ++ sql = ( ++ f"SELECT 1 FROM {options.change_function}" ++ f"(TABLE `{project}.{dataset}.{table_name}`, " ++ f"NULL, NULL) LIMIT 0") ++ request = bigquery.BigqueryJobsInsertRequest( ++ projectId=project, ++ job=bigquery.Job( ++ configuration=bigquery.JobConfiguration( ++ query=bigquery.JobConfigurationQuery( ++ query=sql, ++ useLegacySql=False), ++ dryRun=True))) ++ bq.client.jobs.Insert(request) ++ _LOGGER.info( ++ '[Preflight] %s() access verified for %s:%s.%s', ++ options.change_function, project, dataset, table_name) ++ except Exception as e: ++ raise ValueError( ++ f"Cannot execute {options.change_function}() on " ++ f"'{project}:{dataset}.{table_name}'. " ++ f"Verify the table has change history enabled and the service " ++ f"account has bigquery.jobs.create permission. " ++ f"Error: {e}") from e ++ ++ ++def _check_bq_temp_dataset(project, options): ++ """Verify access to the temp dataset (if specified), or check that ++ datasets.create permission exists for auto-creation.""" ++ try: ++ from apache_beam.io.gcp import bigquery_tools ++ from apache_beam.io.gcp.internal.clients import bigquery ++ from apitools.base.py.exceptions import HttpError ++ except ImportError: ++ _LOGGER.warning( ++ '[Preflight] Skipping BQ temp dataset check: ' ++ 'BigQuery client libraries not available') ++ return ++ ++ if options.temp_dataset: ++ try: ++ bq = bigquery_tools.BigQueryWrapper() ++ bq.client.datasets.Get( ++ bigquery.BigqueryDatasetsGetRequest( ++ projectId=project, datasetId=options.temp_dataset)) ++ _LOGGER.info( ++ '[Preflight] Temp dataset %s:%s exists', ++ project, options.temp_dataset) ++ except HttpError as e: ++ if e.status_code == 404: ++ raise ValueError( ++ f"Temp dataset '{project}:{options.temp_dataset}' not found. " ++ f"Create it or omit --temp_dataset for auto-creation.") from e ++ elif e.status_code == 403: ++ raise ValueError( ++ f"No access to temp dataset '{project}:{options.temp_dataset}'. " ++ f"Verify the service account has " ++ f"bigquery.datasets.get permission.") from e ++ raise ++ ++ # Verify we can write to the temp dataset by doing a dry-run query ++ # with a destination table in it. ++ try: ++ temp_table_ref = bigquery.TableReference( ++ projectId=project, ++ datasetId=options.temp_dataset, ++ tableId='beam_ch_preflight_check') ++ request = bigquery.BigqueryJobsInsertRequest( ++ projectId=project, ++ job=bigquery.Job( ++ configuration=bigquery.JobConfiguration( ++ query=bigquery.JobConfigurationQuery( ++ query='SELECT 1', ++ useLegacySql=False, ++ destinationTable=temp_table_ref, ++ writeDisposition='WRITE_TRUNCATE'), ++ dryRun=True))) ++ bq.client.jobs.Insert(request) ++ _LOGGER.info( ++ '[Preflight] Write access to temp dataset %s:%s verified', ++ project, options.temp_dataset) ++ except Exception as e: ++ raise ValueError( ++ f"Cannot write to temp dataset '{project}:{options.temp_dataset}'. " ++ f"Verify the service account has bigquery.tables.create and " ++ f"bigquery.tables.updateData permissions on this dataset. " ++ f"Error: {e}") from e ++ else: ++ _LOGGER.info( ++ '[Preflight] No --temp_dataset specified; ' ++ 'will auto-create at runtime (requires bigquery.datasets.create)') ++ ++ ++def _check_pubsub_topic(topic_path): ++ """Verify the Pub/Sub topic exists.""" ++ try: ++ from google.cloud import pubsub_v1 ++ from google.api_core.exceptions import NotFound, PermissionDenied ++ except ImportError: ++ _LOGGER.warning( ++ '[Preflight] Skipping Pub/Sub check: ' ++ 'google-cloud-pubsub not available') ++ return ++ ++ try: ++ publisher = pubsub_v1.PublisherClient() ++ publisher.get_topic(topic=topic_path) ++ _LOGGER.info('[Preflight] Pub/Sub topic %s is accessible', topic_path) ++ except NotFound: ++ raise ValueError( ++ f"Pub/Sub topic '{topic_path}' not found. " ++ f"Create it with: gcloud pubsub topics create {topic_path}") ++ except PermissionDenied as e: ++ raise ValueError( ++ f"No permission to access Pub/Sub topic '{topic_path}'. " ++ f"Verify the service account has pubsub.topics.get and " ++ f"pubsub.topics.publish permissions. Error: {e}") from e ++ except Exception as e: ++ _LOGGER.warning( ++ '[Preflight] Could not verify Pub/Sub topic %s: %s', ++ topic_path, e) ++ ++ ++# --------------------------------------------------------------------------- ++# Pipeline construction ++# --------------------------------------------------------------------------- ++ ++ ++def build_pipeline(pipeline, options, metric_spec, detector): ++ """Construct the anomaly monitoring pipeline. ++ ++ Args: ++ pipeline: A beam.Pipeline instance. ++ options: AnomalyMonitorOptions with table, poll_interval_sec, etc. ++ metric_spec: Parsed MetricSpec instance. ++ detector: Parsed anomaly detector instance. ++ ++ Returns: ++ The final PCollection (for testing). ++ """ ++ from bqmonitor.cdc import ReadBigQueryChangeHistory ++ ++ start_time = time.time() - options.start_offset_sec ++ stop_time = ( ++ time.time() + options.duration_sec if options.duration_sec > 0 else None) ++ ++ _LOGGER.info('Anomaly Monitor Pipeline') ++ _LOGGER.info(' Table: %s', options.table) ++ _LOGGER.info(' Detector: %s', type(detector).__name__) ++ _LOGGER.info(' Poll interval: %d sec', options.poll_interval_sec) ++ _LOGGER.info(' Change function: %s', options.change_function) ++ ++ columns = sorted(metric_spec.required_source_columns()) ++ _LOGGER.info(' Columns: %s', columns) ++ ++ # Auto-rename pseudo-columns if they conflict with user column names. ++ change_type_col = 'change_type' ++ change_ts_col = 'change_timestamp' ++ col_set = set(columns) ++ if change_type_col in col_set: ++ change_type_col = '_bqm_change_type' ++ _LOGGER.info( ++ ' Renamed pseudo-column change_type -> %s to avoid conflict', ++ change_type_col) ++ if change_ts_col in col_set: ++ change_ts_col = '_bqm_change_timestamp' ++ _LOGGER.info( ++ ' Renamed pseudo-column change_timestamp -> %s to avoid conflict', ++ change_ts_col) ++ ++ cdc_kwargs = dict( ++ table=options.table, ++ poll_interval_sec=options.poll_interval_sec, ++ start_time=start_time, ++ change_function=options.change_function, ++ buffer_sec=options.buffer_sec, ++ columns=columns, ++ change_type_column=change_type_col, ++ change_timestamp_column=change_ts_col, ++ decompress_shards=( ++ options.decompress_shards if options.decompress_shards > 0 ++ else None)) ++ if stop_time is not None: ++ cdc_kwargs['stop_time'] = stop_time ++ if options.temp_dataset: ++ cdc_kwargs['temp_dataset'] = options.temp_dataset ++ ++ rows = pipeline | 'ReadCDC' >> ReadBigQueryChangeHistory(**cdc_kwargs) ++ fanout_strategy = FanoutStrategy(options.fanout_strategy) ++ metrics = rows | 'ComputeMetric' >> ComputeMetric( ++ metric_spec, fanout_strategy=fanout_strategy, fanout=options.fanout) ++ ++ # Rewindow into GlobalWindows so the anomaly detector sees the full ++ # stream of window results as a time series, not isolated per-window. ++ from apache_beam.transforms.window import GlobalWindows ++ global_metrics = metrics | 'Rewindow' >> beam.WindowInto(GlobalWindows()) ++ ++ if isinstance(detector, _ThresholdAlert): ++ anomalies = global_metrics | 'DetectAnomalies' >> beam.ParDo(detector) ++ else: ++ anomalies = global_metrics | 'DetectAnomalies' >> AnomalyDetection(detector) ++ ++ if options.log_all_results.lower() == 'true': ++ _ = anomalies | 'LogResults' >> beam.ParDo(_LogAnomalyResult()) ++ ++ # Publish anomalies (label == 1) to Pub/Sub. ++ project, _, _ = _parse_table_ref(options.table) ++ topic_path = f'projects/{project}/topics/{options.topic}' ++ ++ _ = ( ++ anomalies ++ | 'FormatAnomalies' >> beam.ParDo(_FormatAnomalyAsJson()) ++ | 'WriteToPubSub' >> WriteToPubSub(topic=topic_path)) ++ ++ # Write all results to a BigQuery sink table (if configured). ++ if options.sink_table: ++ sink_table = options.sink_table.replace(':', '.') ++ _ = ( ++ anomalies ++ | 'FormatForBQ' >> beam.ParDo(_FormatResultForBQ()) ++ | 'WriteSink' >> WriteToBigQuery( ++ table=sink_table, ++ method=options.write_method, ++ schema=_SINK_SCHEMA, ++ create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, ++ write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)) ++ ++ return anomalies ++ ++ ++def run(argv=None): ++ """Main entry point.""" ++ options = PipelineOptions(argv) ++ monitor_options = options.view_as(AnomalyMonitorOptions) ++ ++ # Validate required options. ++ for required_opt in ('table', 'metric_spec', 'detector_spec', 'topic'): ++ if getattr(monitor_options, required_opt) is None: ++ raise ValueError(f'--{required_opt} is required') ++ ++ # Validate table format. ++ _parse_table_ref(monitor_options.table) ++ ++ # Parse specs early so errors surface before pipeline construction. ++ metric_spec = _parse_metric_spec(monitor_options.metric_spec) ++ detector = _parse_detector_spec(monitor_options.detector_spec) ++ ++ # Check GCP resources are accessible. ++ _preflight_checks(monitor_options) ++ ++ options.view_as(SetupOptions).save_main_session = True ++ ++ with beam.Pipeline(options=options) as p: ++ build_pipeline(p, monitor_options, metric_spec, detector) ++ ++ ++if __name__ == '__main__': ++ logging.getLogger().setLevel(logging.INFO) ++ run() +diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py +new file mode 100644 +index 000000000..3cc9e00d2 +--- /dev/null ++++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py +@@ -0,0 +1,192 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Safe expression evaluator for metric computation. ++ ++Parses a Python expression string, validates it uses only allowed ++constructs, compiles it, and produces a callable that evaluates the ++expression against a field context dict. ++ ++Example usage:: ++ ++ from bqmonitor.safe_eval import Expr ++ ++ expr = Expr.from_string("clicks / impressions") ++ expr({'clicks': 50, 'impressions': 1000}) # 0.05 ++ ++ expr = Expr.from_string("1 if status == 'success' else 0") ++ expr({'status': 'success'}) # 1 ++ ++Allowed constructs: field names (bare names), literals (int, float, str), ++arithmetic (``+, -, *, /, //, %, **``), comparisons (``==, !=, <, <=, >, >=``), ++boolean logic (``and, or, not``), unary negation, ``if/else``, and ++safe builtins (``abs, min, max, round``). ++""" ++ ++import ast ++ ++# --- AST whitelist --- ++ ++_ALLOWED_BINOPS = ( ++ ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow) ++ ++_ALLOWED_CMPOPS = (ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE) ++ ++_SAFE_BUILTINS = {"abs": abs, "min": min, "max": max, "round": round} ++ ++ ++def _validate_ast(node): ++ """Recursively validate that an AST node uses only allowed constructs.""" ++ if isinstance(node, ast.Name): ++ return ++ ++ if isinstance(node, ast.Constant): ++ if not isinstance(node.value, (int, float, str)): ++ raise ValueError( ++ f"Unsupported literal type: {type(node.value).__name__}. " ++ f"Only int, float, and str literals are supported.") ++ return ++ ++ if isinstance(node, ast.UnaryOp): ++ if not isinstance(node.op, (ast.USub, ast.Not)): ++ raise ValueError( ++ f"Unsupported unary operator: {type(node.op).__name__}. " ++ f"Only negation (-) and not are supported.") ++ _validate_ast(node.operand) ++ return ++ ++ if isinstance(node, ast.BinOp): ++ if not isinstance(node.op, _ALLOWED_BINOPS): ++ raise ValueError( ++ f"Unsupported binary operator: {type(node.op).__name__}. " ++ f"Supported: +, -, *, /, //, %, **") ++ _validate_ast(node.left) ++ _validate_ast(node.right) ++ return ++ ++ if isinstance(node, ast.BoolOp): ++ # and / or ++ for value in node.values: ++ _validate_ast(value) ++ return ++ ++ if isinstance(node, ast.Compare): ++ if len(node.ops) != 1 or len(node.comparators) != 1: ++ raise ValueError( ++ "Chained comparisons not supported (e.g., a < b < c). " ++ "Use (a < b) and separate expressions instead.") ++ if not isinstance(node.ops[0], _ALLOWED_CMPOPS): ++ raise ValueError( ++ f"Unsupported comparison: {type(node.ops[0]).__name__}. " ++ f"Supported: ==, !=, <, <=, >, >=") ++ _validate_ast(node.left) ++ _validate_ast(node.comparators[0]) ++ return ++ ++ if isinstance(node, ast.IfExp): ++ _validate_ast(node.test) ++ _validate_ast(node.body) ++ _validate_ast(node.orelse) ++ return ++ ++ if isinstance(node, ast.Call): ++ if not (isinstance(node.func, ast.Name) ++ and node.func.id in _SAFE_BUILTINS): ++ name = node.func.id if isinstance(node.func, ast.Name) else ast.dump( ++ node.func) ++ raise ValueError( ++ f"Unsupported function: {name}. " ++ f"Supported: {', '.join(sorted(_SAFE_BUILTINS))}.") ++ if node.keywords: ++ raise ValueError("Keyword arguments not supported in function calls.") ++ for arg in node.args: ++ _validate_ast(arg) ++ return ++ ++ raise ValueError( ++ f"Unsupported expression: {ast.dump(node)}. " ++ f"Only field names, literals, arithmetic (+,-,*,/,//,%,**), " ++ f"comparisons (==,!=,<,<=,>,>=), boolean logic (and, or, not), " ++ f"if/else, and functions ({', '.join(sorted(_SAFE_BUILTINS))}) " ++ f"are supported.") ++ ++ ++def _collect_field_refs(node): ++ """Collect all field names referenced in an AST (excluding builtins).""" ++ return frozenset( ++ child.id for child in ast.walk(node) ++ if isinstance(child, ast.Name) and child.id not in _SAFE_BUILTINS) ++ ++ ++class Expr: ++ """A validated, compiled expression callable. ++ ++ Parses a Python expression string, validates it uses only safe ++ constructs, and compiles it into a callable. The compiled expression ++ is evaluated with restricted builtins (no access to ``import``, ++ ``open``, ``exec``, etc.). ++ ++ Args: ++ text: A Python expression string. ++ ++ Raises: ++ ValueError: If the expression uses unsupported Python constructs. ++ SyntaxError: If the string is not valid Python syntax. ++ """ ++ def __init__(self, text): ++ self._text = text ++ tree = ast.parse(text, mode='eval') ++ _validate_ast(tree.body) ++ self._code = compile(tree, '', 'eval') ++ self._refs = _collect_field_refs(tree.body) ++ ++ def __call__(self, context): ++ """Evaluate the expression against a dict of field values.""" ++ env = dict(_SAFE_BUILTINS) ++ env.update(context) ++ return eval(self._code, {"__builtins__": {}}, env) ++ ++ def field_refs(self): ++ """Return the set of field names referenced by this expression.""" ++ return set(self._refs) ++ ++ @staticmethod ++ def from_string(text): ++ """Parse a Python expression string into a compiled Expr callable. ++ ++ Examples:: ++ ++ Expr.from_string("clicks / impressions") ++ Expr.from_string("1 if status == 'success' else 0") ++ Expr.from_string("(a + b) / total") ++ """ ++ return Expr(text) ++ ++ def __str__(self): ++ return self._text ++ ++ def __repr__(self): ++ return f"Expr({self._text!r})" ++ ++ def __eq__(self, other): ++ return isinstance(other, Expr) and self._text == other._text ++ ++ def __hash__(self): ++ return hash(self._text) ++ ++ def __reduce__(self): ++ """Pickle support: store text, recompile on unpickle.""" ++ return (Expr, (self._text, )) +diff --git a/python/src/test/python/bigquery-anomaly-detection/__init__.py b/python/src/test/python/bigquery-anomaly-detection/__init__.py +new file mode 100644 +index 000000000..e69de29bb +diff --git a/python/src/test/python/bigquery-anomaly-detection/metric_test.py b/python/src/test/python/bigquery-anomaly-detection/metric_test.py +new file mode 100644 +index 000000000..bd6fcd244 +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/metric_test.py +@@ -0,0 +1,243 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Unit tests for bqmonitor.metric.""" ++ ++import json ++import logging ++import unittest ++ ++logging.basicConfig(level=logging.INFO) ++ ++from bqmonitor.metric import AggOp ++from bqmonitor.metric import AggregationSpec ++from bqmonitor.metric import DerivedField ++from bqmonitor.metric import MeasureSpec ++from bqmonitor.metric import MetricSpec ++from bqmonitor.metric import WindowSpec ++from bqmonitor.metric import WindowType ++from bqmonitor.safe_eval import Expr ++ ++ ++class MetricSpecValidationTest(unittest.TestCase): ++ """Tests for MetricSpec validation.""" ++ ++ def _simple_spec(self, **kwargs): ++ defaults = dict( ++ aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=60), ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, ++ alias='total')], ++ )) ++ defaults.update(kwargs) ++ return MetricSpec(**defaults) ++ ++ def test_simple_spec_valid(self): ++ spec = self._simple_spec() ++ self.assertEqual(len(spec.aggregation.measures), 1) ++ ++ def test_no_measures_raises(self): ++ # @specifiable uses lazy init; access an attribute to trigger validation. ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec(aggregation=AggregationSpec(measures=[])) ++ _ = spec.aggregation ++ self.assertIn('at least one measure', str(ctx.exception)) ++ ++ def test_multiple_measures_without_combiner_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='x'), ++ MeasureSpec(field='b', agg=AggOp.COUNT, alias='y'), ++ ])) ++ _ = spec.aggregation ++ self.assertIn('measure_combiner is required', str(ctx.exception)) ++ ++ def test_multiple_measures_with_combiner(self): ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec(field='a', agg=AggOp.COUNT, alias='impressions'), ++ ]), ++ measure_combiner=Expr('clicks / impressions')) ++ self.assertIsNotNone(spec.measure_combiner) ++ ++ def test_combiner_unknown_field_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), ++ ]), ++ measure_combiner=Expr('clicks / impressions')) ++ _ = spec.aggregation ++ self.assertIn('unknown fields', str(ctx.exception)) ++ ++ def test_sliding_without_period_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ spec = MetricSpec(aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.SLIDING, size_seconds=60), ++ measures=[MeasureSpec(field='a', agg=AggOp.SUM, alias='x')])) ++ _ = spec.aggregation ++ self.assertIn('period_seconds', str(ctx.exception)) ++ ++ def test_sliding_with_period(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ window=WindowSpec( ++ type=WindowType.SLIDING, size_seconds=60, period_seconds=10), ++ measures=[MeasureSpec(field='a', agg=AggOp.SUM, alias='x')])) ++ self.assertEqual(spec.aggregation.window.period_seconds, 10) ++ ++ ++class MetricSpecRequiredColumnsTest(unittest.TestCase): ++ """Tests for required_source_columns().""" ++ ++ def test_simple_sum(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) ++ self.assertEqual(spec.required_source_columns(), {'amount'}) ++ ++ def test_count_excludes_field(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[MeasureSpec(field='x', agg=AggOp.COUNT, alias='cnt')])) ++ self.assertEqual(spec.required_source_columns(), set()) ++ ++ def test_group_by_included(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ group_by=['region', 'product'], ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) ++ self.assertEqual( ++ spec.required_source_columns(), {'region', 'product', 'amount'}) ++ ++ def test_derived_field_references(self): ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ measures=[MeasureSpec( ++ field='is_success', agg=AggOp.SUM, alias='successes')]), ++ derived_fields=[ ++ DerivedField( ++ name='is_success', ++ expression=Expr("1 if status == 'ok' else 0")) ++ ]) ++ # 'is_success' is derived, so excluded; 'status' is a source ref. ++ self.assertEqual(spec.required_source_columns(), {'status'}) ++ ++ def test_ctr_metric(self): ++ spec = MetricSpec( ++ aggregation=AggregationSpec( ++ group_by=['campaign_type'], ++ measures=[ ++ MeasureSpec(field='is_click', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec( ++ field='is_click', agg=AggOp.COUNT, alias='impressions'), ++ ]), ++ measure_combiner=Expr('clicks / impressions')) ++ # is_click (from SUM), campaign_type (from group_by). ++ # COUNT's field is excluded. ++ self.assertEqual( ++ spec.required_source_columns(), {'is_click', 'campaign_type'}) ++ ++ ++class MetricSpecFromDictTest(unittest.TestCase): ++ """Tests for MetricSpec.from_dict() deserialization.""" ++ ++ def test_simple(self): ++ d = { ++ 'aggregation': { ++ 'window': {'type': 'fixed', 'size_seconds': 60}, ++ 'measures': [ ++ {'field': 'amount', 'agg': 'SUM', 'alias': 'total'}], ++ }} ++ spec = MetricSpec.from_dict(d) ++ self.assertEqual(spec.aggregation.window.type, WindowType.FIXED) ++ self.assertEqual(spec.aggregation.window.size_seconds, 60) ++ self.assertEqual(len(spec.aggregation.measures), 1) ++ self.assertEqual(spec.aggregation.measures[0].agg, AggOp.SUM) ++ ++ def test_with_combiner(self): ++ d = { ++ 'aggregation': { ++ 'measures': [ ++ {'field': 'x', 'agg': 'SUM', 'alias': 'clicks'}, ++ {'field': 'x', 'agg': 'COUNT', 'alias': 'impressions'}], ++ }, ++ 'measure_combiner': {'expression': 'clicks / impressions'}, ++ } ++ spec = MetricSpec.from_dict(d) ++ self.assertIsNotNone(spec.measure_combiner) ++ self.assertEqual(spec.measure_combiner.field_refs(), {'clicks', 'impressions'}) ++ ++ def test_with_derived_fields(self): ++ d = { ++ 'aggregation': { ++ 'measures': [ ++ {'field': 'is_ok', 'agg': 'SUM', 'alias': 'ok_count'}], ++ }, ++ 'derived_fields': [ ++ {'name': 'is_ok', 'expression': "1 if status == 'ok' else 0"}], ++ } ++ spec = MetricSpec.from_dict(d) ++ self.assertEqual(len(spec.derived_fields), 1) ++ self.assertEqual(spec.derived_fields[0].name, 'is_ok') ++ ++ def test_roundtrip_json(self): ++ """from_dict(to_dict(spec)) should produce an equivalent spec.""" ++ original = MetricSpec( ++ aggregation=AggregationSpec( ++ window=WindowSpec( ++ type=WindowType.SLIDING, size_seconds=300, period_seconds=60), ++ group_by=['region'], ++ measures=[ ++ MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), ++ MeasureSpec(field='a', agg=AggOp.COUNT, alias='impressions'), ++ ]), ++ measure_combiner=Expr('clicks / impressions'), ++ name='ctr') ++ d = original.to_dict() ++ # Verify JSON-serializable. ++ json_str = json.dumps(d) ++ restored = MetricSpec.from_dict(json.loads(json_str)) ++ self.assertEqual(restored.name, 'ctr') ++ self.assertEqual(restored.aggregation.window.type, WindowType.SLIDING) ++ self.assertEqual(restored.aggregation.window.period_seconds, 60) ++ self.assertEqual(len(restored.aggregation.measures), 2) ++ self.assertEqual( ++ restored.required_source_columns(), {'a', 'region'}) ++ ++ ++class MetricSpecToDictTest(unittest.TestCase): ++ """Tests for MetricSpec.to_dict() serialization.""" ++ ++ def test_simple(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ window=WindowSpec(type=WindowType.FIXED, size_seconds=60), ++ measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) ++ d = spec.to_dict() ++ self.assertEqual(d['aggregation']['window']['type'], 'fixed') ++ self.assertEqual(d['aggregation']['measures'][0]['agg'], 'SUM') ++ ++ def test_excludes_optional_none(self): ++ spec = MetricSpec(aggregation=AggregationSpec( ++ measures=[MeasureSpec(field='x', agg=AggOp.SUM, alias='y')])) ++ d = spec.to_dict() ++ self.assertNotIn('derived_fields', d) ++ self.assertNotIn('measure_combiner', d) ++ self.assertNotIn('name', d) ++ ++ ++if __name__ == '__main__': ++ unittest.main() +diff --git a/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py b/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py +new file mode 100644 +index 000000000..3f654893d +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py +@@ -0,0 +1,287 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Unit tests for bqmonitor.pipeline helpers.""" ++ ++import json ++import logging ++import unittest ++ ++logging.basicConfig(level=logging.INFO) ++ ++import apache_beam as beam ++from apache_beam.ml.anomaly.base import AnomalyPrediction ++from apache_beam.ml.anomaly.base import AnomalyResult ++ ++from bqmonitor.pipeline import _FormatAnomalyAsJson ++from bqmonitor.pipeline import _FormatResultForBQ ++from bqmonitor.pipeline import _parse_detector_spec ++from bqmonitor.pipeline import _parse_table_ref ++from bqmonitor.pipeline import _ThresholdAlert ++from bqmonitor.pipeline import _unpack_result ++ ++ ++class ParseTableRefTest(unittest.TestCase): ++ """Tests for _parse_table_ref().""" ++ ++ def test_colon_format(self): ++ p, d, t = _parse_table_ref('my-project:my_dataset.my_table') ++ self.assertEqual(p, 'my-project') ++ self.assertEqual(d, 'my_dataset') ++ self.assertEqual(t, 'my_table') ++ ++ def test_dot_format(self): ++ p, d, t = _parse_table_ref('my-project.my_dataset.my_table') ++ self.assertEqual(p, 'my-project') ++ self.assertEqual(d, 'my_dataset') ++ self.assertEqual(t, 'my_table') ++ ++ def test_invalid_format_raises(self): ++ with self.assertRaises(ValueError): ++ _parse_table_ref('not_valid') ++ ++ def test_empty_raises(self): ++ with self.assertRaises(ValueError): ++ _parse_table_ref('') ++ ++ ++class UnpackResultTest(unittest.TestCase): ++ """Tests for _unpack_result().""" ++ ++ def test_keyed(self): ++ result = object() ++ key, r = _unpack_result(('mykey', result)) ++ self.assertEqual(key, 'mykey') ++ self.assertIs(r, result) ++ ++ def test_unkeyed(self): ++ result = object() ++ key, r = _unpack_result(result) ++ self.assertIsNone(key) ++ self.assertIs(r, result) ++ ++ ++class ParseDetectorSpecTest(unittest.TestCase): ++ """Tests for _parse_detector_spec().""" ++ ++ def test_zscore(self): ++ detector = _parse_detector_spec('{"type":"ZScore"}') ++ self.assertEqual(type(detector).__name__, 'ZScore') ++ ++ def test_iqr(self): ++ detector = _parse_detector_spec('{"type":"IQR"}') ++ self.assertEqual(type(detector).__name__, 'IQR') ++ ++ def test_robust_zscore(self): ++ detector = _parse_detector_spec('{"type":"RobustZScore"}') ++ self.assertEqual(type(detector).__name__, 'RobustZScore') ++ ++ def test_threshold(self): ++ detector = _parse_detector_spec( ++ '{"type":"Threshold","expression":"value >= 100"}') ++ self.assertIsInstance(detector, _ThresholdAlert) ++ ++ def test_threshold_missing_expression_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec('{"type":"Threshold"}') ++ self.assertIn('expression', str(ctx.exception)) ++ ++ def test_threshold_invalid_expression_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec( ++ '{"type":"Threshold","expression":"import os"}') ++ self.assertIn('Invalid threshold expression', str(ctx.exception)) ++ ++ def test_unknown_type_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec('{"type":"Unknown"}') ++ self.assertIn('Unknown', str(ctx.exception)) ++ ++ def test_invalid_json_raises(self): ++ with self.assertRaises(ValueError) as ctx: ++ _parse_detector_spec('{bad json}') ++ self.assertIn('Invalid JSON', str(ctx.exception)) ++ ++ def test_missing_type_raises(self): ++ with self.assertRaises(ValueError): ++ _parse_detector_spec('{"config":{}}') ++ ++ def test_zscore_with_threshold(self): ++ spec = json.dumps({ ++ 'type': 'ZScore', ++ 'config': { ++ 'threshold_criterion': { ++ 'type': 'FixedThreshold', ++ 'config': {'cutoff': 10}}}}) ++ detector = _parse_detector_spec(spec) ++ self.assertEqual(type(detector).__name__, 'ZScore') ++ ++ ++class ThresholdAlertTest(unittest.TestCase): ++ """Tests for _ThresholdAlert DoFn.""" ++ ++ def _make_row(self, value): ++ return beam.Row(value=value, window_start=0.0, window_end=1.0) ++ ++ def _run_dofn(self, expression, element): ++ dofn = _ThresholdAlert(expression) ++ dofn.setup() ++ return list(dofn.process(element)) ++ ++ def test_above_threshold(self): ++ results = self._run_dofn('value >= 100', self._make_row(500.0)) ++ self.assertEqual(len(results), 1) ++ result = results[0] ++ self.assertIsInstance(result, AnomalyResult) ++ self.assertEqual(result.predictions[0].label, 1) ++ self.assertIsNone(result.predictions[0].score) ++ self.assertEqual( ++ result.predictions[0].model_id, 'Threshold(value >= 100)') ++ ++ def test_below_threshold(self): ++ results = self._run_dofn('value >= 100', self._make_row(50.0)) ++ self.assertEqual(len(results), 1) ++ self.assertEqual(results[0].predictions[0].label, 0) ++ ++ def test_keyed_element(self): ++ row = self._make_row(200.0) ++ results = self._run_dofn('value >= 100', ('mykey', row)) ++ self.assertEqual(len(results), 1) ++ key, result = results[0] ++ self.assertEqual(key, 'mykey') ++ self.assertEqual(result.predictions[0].label, 1) ++ ++ def test_range_expression(self): ++ dofn = _ThresholdAlert('value > 100 or value < -100') ++ dofn.setup() ++ ++ # Above range. ++ results = list(dofn.process(self._make_row(200.0))) ++ self.assertEqual(results[0].predictions[0].label, 1) ++ ++ # Below range. ++ results = list(dofn.process(self._make_row(-200.0))) ++ self.assertEqual(results[0].predictions[0].label, 1) ++ ++ # Within range. ++ results = list(dofn.process(self._make_row(50.0))) ++ self.assertEqual(results[0].predictions[0].label, 0) ++ ++ def test_less_than_threshold(self): ++ results = self._run_dofn('value <= 0.01', self._make_row(0.005)) ++ self.assertEqual(results[0].predictions[0].label, 1) ++ ++ results = self._run_dofn('value <= 0.01', self._make_row(0.5)) ++ self.assertEqual(results[0].predictions[0].label, 0) ++ ++ ++class FormatAnomalyAsJsonTest(unittest.TestCase): ++ """Tests for _FormatAnomalyAsJson DoFn.""" ++ ++ def _make_result(self, label, value=42.0, score=5.0, model_id='TestModel'): ++ row = beam.Row(value=value, window_start=1000.0, window_end=1001.0) ++ prediction = AnomalyPrediction( ++ model_id=model_id, score=score, label=label) ++ return AnomalyResult(example=row, predictions=[prediction]) ++ ++ def test_outlier_emits_json(self): ++ dofn = _FormatAnomalyAsJson() ++ results = list(dofn.process(self._make_result(label=1))) ++ self.assertEqual(len(results), 1) ++ payload = json.loads(results[0]) ++ self.assertIn('Anomaly detected', payload['event_description']) ++ self.assertEqual(payload['agent_id'], 'TestModel') ++ ++ def test_normal_emits_nothing(self): ++ dofn = _FormatAnomalyAsJson() ++ results = list(dofn.process(self._make_result(label=0))) ++ self.assertEqual(len(results), 0) ++ ++ def test_warmup_emits_nothing(self): ++ dofn = _FormatAnomalyAsJson() ++ results = list(dofn.process(self._make_result(label=-2))) ++ self.assertEqual(len(results), 0) ++ ++ def test_keyed_outlier_includes_key(self): ++ dofn = _FormatAnomalyAsJson() ++ result = self._make_result(label=1) ++ outputs = list(dofn.process(('campaign_search', result))) ++ self.assertEqual(len(outputs), 1) ++ payload = json.loads(outputs[0]) ++ self.assertEqual(payload['key'], 'campaign_search') ++ ++ def test_threshold_model_id(self): ++ dofn = _FormatAnomalyAsJson() ++ result = self._make_result( ++ label=1, model_id='Threshold(value >= 100)') ++ outputs = list(dofn.process(result)) ++ payload = json.loads(outputs[0]) ++ self.assertEqual(payload['agent_id'], 'Threshold(value >= 100)') ++ ++ ++class FormatResultForBQTest(unittest.TestCase): ++ """Tests for _FormatResultForBQ DoFn.""" ++ ++ def _make_result(self, label, value=42.0, score=5.0): ++ row = beam.Row(value=value, window_start=1000.0, window_end=1001.0) ++ prediction = AnomalyPrediction( ++ model_id='TestModel', score=score, label=label) ++ return AnomalyResult(example=row, predictions=[prediction]) ++ ++ def test_outlier_row(self): ++ dofn = _FormatResultForBQ() ++ results = list(dofn.process(self._make_result(label=1, value=99.0, ++ score=4.5))) ++ self.assertEqual(len(results), 1) ++ row = results[0] ++ self.assertAlmostEqual(row['value'], 99.0) ++ self.assertAlmostEqual(row['score'], 4.5) ++ self.assertEqual(row['label'], 1) ++ self.assertIn('window_start', row) ++ self.assertIn('window_end', row) ++ self.assertNotIn('key', row) ++ ++ def test_normal_row(self): ++ dofn = _FormatResultForBQ() ++ results = list(dofn.process(self._make_result(label=0))) ++ self.assertEqual(len(results), 1) ++ self.assertEqual(results[0]['label'], 0) ++ ++ def test_warmup_row(self): ++ dofn = _FormatResultForBQ() ++ results = list(dofn.process(self._make_result(label=-2))) ++ self.assertEqual(len(results), 1) ++ self.assertEqual(results[0]['label'], -2) ++ ++ def test_keyed_row_includes_key(self): ++ dofn = _FormatResultForBQ() ++ result = self._make_result(label=1) ++ outputs = list(dofn.process(('campaign_search', result))) ++ self.assertEqual(len(outputs), 1) ++ self.assertEqual(outputs[0]['key'], 'campaign_search') ++ ++ def test_none_score(self): ++ row = beam.Row(value=10.0, window_start=0.0, window_end=1.0) ++ prediction = AnomalyPrediction( ++ model_id='Test', score=None, label=0) ++ result = AnomalyResult(example=row, predictions=[prediction]) ++ dofn = _FormatResultForBQ() ++ outputs = list(dofn.process(result)) ++ self.assertIsNone(outputs[0]['score']) ++ ++ ++if __name__ == '__main__': ++ unittest.main() +diff --git a/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt b/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt +new file mode 100644 +index 000000000..57726014e +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt +@@ -0,0 +1 @@ ++apache-beam[gcp,test] +diff --git a/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py b/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py +new file mode 100644 +index 000000000..974b10511 +--- /dev/null ++++ b/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py +@@ -0,0 +1,244 @@ ++# ++# Copyright (C) 2026 Google LLC ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may not ++# use this file except in compliance with the License. You may obtain a copy of ++# the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations under ++# the License. ++# ++ ++"""Unit tests for bqmonitor.safe_eval.""" ++ ++import logging ++import unittest ++ ++logging.basicConfig(level=logging.INFO) ++ ++from bqmonitor.safe_eval import Expr ++ ++ ++class ExprArithmeticTest(unittest.TestCase): ++ """Tests for arithmetic operations.""" ++ ++ def test_addition(self): ++ self.assertEqual(Expr('a + b')({'a': 3, 'b': 4}), 7) ++ ++ def test_subtraction(self): ++ self.assertEqual(Expr('a - b')({'a': 10, 'b': 3}), 7) ++ ++ def test_multiplication(self): ++ self.assertEqual(Expr('a * b')({'a': 5, 'b': 6}), 30) ++ ++ def test_division(self): ++ self.assertAlmostEqual(Expr('a / b')({'a': 10, 'b': 3}), 10 / 3) ++ ++ def test_floor_division(self): ++ self.assertEqual(Expr('a // b')({'a': 10, 'b': 3}), 3) ++ ++ def test_modulo(self): ++ self.assertEqual(Expr('a % b')({'a': 10, 'b': 3}), 1) ++ ++ def test_power(self): ++ self.assertEqual(Expr('x ** 2')({'x': 5}), 25) ++ ++ def test_negation(self): ++ self.assertEqual(Expr('-x')({'x': 7}), -7) ++ ++ def test_parentheses(self): ++ self.assertEqual(Expr('(a + b) * c')({'a': 2, 'b': 3, 'c': 4}), 20) ++ ++ ++class ExprComparisonTest(unittest.TestCase): ++ """Tests for comparison operations.""" ++ ++ def test_eq(self): ++ self.assertTrue(Expr('x == 1')({'x': 1})) ++ self.assertFalse(Expr('x == 1')({'x': 2})) ++ ++ def test_neq(self): ++ self.assertTrue(Expr('x != 1')({'x': 2})) ++ ++ def test_lt(self): ++ self.assertTrue(Expr('x < 5')({'x': 3})) ++ self.assertFalse(Expr('x < 5')({'x': 5})) ++ ++ def test_lte(self): ++ self.assertTrue(Expr('x <= 5')({'x': 5})) ++ ++ def test_gt(self): ++ self.assertTrue(Expr('x > 5')({'x': 6})) ++ ++ def test_gte(self): ++ self.assertTrue(Expr('x >= 5')({'x': 5})) ++ ++ def test_string_comparison(self): ++ self.assertTrue(Expr("s == 'ok'")({'s': 'ok'})) ++ self.assertFalse(Expr("s == 'ok'")({'s': 'fail'})) ++ ++ ++class ExprBooleanTest(unittest.TestCase): ++ """Tests for boolean logic.""" ++ ++ def test_and(self): ++ self.assertTrue(Expr('a > 0 and b > 0')({'a': 1, 'b': 1})) ++ self.assertFalse(Expr('a > 0 and b > 0')({'a': 1, 'b': -1})) ++ ++ def test_or(self): ++ self.assertTrue(Expr('a > 0 or b > 0')({'a': -1, 'b': 1})) ++ self.assertFalse(Expr('a > 0 or b > 0')({'a': -1, 'b': -1})) ++ ++ def test_not(self): ++ self.assertTrue(Expr('not failed')({'failed': False})) ++ self.assertFalse(Expr('not failed')({'failed': True})) ++ ++ def test_compound(self): ++ e = Expr("x > 0 and not disabled or override == 'yes'") ++ self.assertTrue(e({'x': 1, 'disabled': False, 'override': 'no'})) ++ self.assertTrue(e({'x': -1, 'disabled': True, 'override': 'yes'})) ++ self.assertFalse(e({'x': -1, 'disabled': True, 'override': 'no'})) ++ ++ ++class ExprIfElseTest(unittest.TestCase): ++ """Tests for conditional expressions.""" ++ ++ def test_if_else(self): ++ e = Expr("1 if status == 'ok' else 0") ++ self.assertEqual(e({'status': 'ok'}), 1) ++ self.assertEqual(e({'status': 'fail'}), 0) ++ ++ def test_if_else_with_bool(self): ++ e = Expr("1 if a > 0 and b > 0 else 0") ++ self.assertEqual(e({'a': 1, 'b': 1}), 1) ++ self.assertEqual(e({'a': 1, 'b': -1}), 0) ++ ++ ++class ExprBuiltinsTest(unittest.TestCase): ++ """Tests for safe builtin functions.""" ++ ++ def test_abs(self): ++ self.assertEqual(Expr('abs(x)')({'x': -7}), 7) ++ self.assertEqual(Expr('abs(x)')({'x': 7}), 7) ++ ++ def test_min(self): ++ self.assertEqual(Expr('min(a, b)')({'a': 3, 'b': 5}), 3) ++ ++ def test_max(self): ++ self.assertEqual(Expr('max(a, b)')({'a': 3, 'b': 5}), 5) ++ ++ def test_max_with_literal(self): ++ self.assertEqual(Expr('max(x, 1)')({'x': 0}), 1) ++ ++ def test_round(self): ++ self.assertAlmostEqual(Expr('round(x, 2)')({'x': 3.14159}), 3.14) ++ ++ def test_nested_builtins(self): ++ self.assertEqual(Expr('max(abs(a), abs(b))')({'a': -5, 'b': 3}), 5) ++ ++ ++class ExprFieldRefsTest(unittest.TestCase): ++ """Tests for field_refs() extraction.""" ++ ++ def test_simple(self): ++ self.assertEqual(Expr('a + b').field_refs(), {'a', 'b'}) ++ ++ def test_excludes_builtins(self): ++ self.assertEqual( ++ Expr('max(clicks, 1) / abs(total)').field_refs(), ++ {'clicks', 'total'}) ++ ++ def test_if_else_refs(self): ++ self.assertEqual( ++ Expr("1 if status == 'ok' else 0").field_refs(), {'status'}) ++ ++ def test_no_refs(self): ++ self.assertEqual(Expr('1 + 2').field_refs(), set()) ++ ++ ++class ExprValidationTest(unittest.TestCase): ++ """Tests for expression validation / rejection.""" ++ ++ def test_rejects_import(self): ++ with self.assertRaises((ValueError, SyntaxError)): ++ Expr('__import__("os")') ++ ++ def test_rejects_unknown_function(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('eval("bad")') ++ self.assertIn('eval', str(ctx.exception)) ++ ++ def test_rejects_attribute_access(self): ++ with self.assertRaises(ValueError): ++ Expr('x.__class__') ++ ++ def test_rejects_subscript(self): ++ with self.assertRaises(ValueError): ++ Expr('x[0]') ++ ++ def test_rejects_lambda(self): ++ with self.assertRaises((ValueError, SyntaxError)): ++ Expr('lambda x: x') ++ ++ def test_rejects_chained_comparison(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('a < b < c') ++ self.assertIn('Chained', str(ctx.exception)) ++ ++ def test_rejects_keyword_args(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('round(x, ndigits=2)') ++ self.assertIn('Keyword', str(ctx.exception)) ++ ++ def test_rejects_unsupported_literal(self): ++ with self.assertRaises(ValueError) as ctx: ++ Expr('b"bytes"') ++ self.assertIn('bytes', str(ctx.exception)) ++ ++ ++class ExprPickleTest(unittest.TestCase): ++ """Tests for pickle/unpickle support.""" ++ ++ def test_roundtrip(self): ++ import pickle ++ original = Expr('a + b') ++ restored = pickle.loads(pickle.dumps(original)) ++ self.assertEqual(original, restored) ++ self.assertEqual(restored({'a': 1, 'b': 2}), 3) ++ ++ ++class ExprRealWorldTest(unittest.TestCase): ++ """Tests using realistic metric expressions.""" ++ ++ def test_ctr(self): ++ e = Expr('clicks / impressions') ++ self.assertAlmostEqual(e({'clicks': 50, 'impressions': 1000}), 0.05) ++ ++ def test_safe_ctr(self): ++ e = Expr('clicks / max(impressions, 1)') ++ self.assertEqual(e({'clicks': 0, 'impressions': 0}), 0.0) ++ ++ def test_derived_field(self): ++ e = Expr("1 if status == 'success' else 0") ++ self.assertEqual(e({'status': 'success'}), 1) ++ self.assertEqual(e({'status': 'error'}), 0) ++ ++ def test_threshold_expression(self): ++ e = Expr('value >= 100') ++ self.assertTrue(e({'value': 500})) ++ self.assertFalse(e({'value': 50})) ++ ++ def test_range_threshold(self): ++ e = Expr('value > 100 or value < -100') ++ self.assertTrue(e({'value': 200})) ++ self.assertTrue(e({'value': -200})) ++ self.assertFalse(e({'value': 50})) ++ ++ ++if __name__ == '__main__': ++ unittest.main() diff --git a/python/README_BigQuery_Anomaly_Detection.md b/python/README_BigQuery_Anomaly_Detection.md new file mode 100644 index 0000000000..9e0fdb7270 --- /dev/null +++ b/python/README_BigQuery_Anomaly_Detection.md @@ -0,0 +1,328 @@ + +BigQuery Anomaly Detection (Experimental) +--- +> **Note:** This template is experimental and may change without notice. + +A streaming Dataflow Flex Template that monitors a BigQuery table for anomalies +in real time. The pipeline reads CDC (Change Data Capture) data from BigQuery, +computes a configurable windowed metric, runs statistical anomaly detection, and +publishes detected anomalies to a Pub/Sub topic. + +Supported anomaly detectors: **ZScore**, **IQR**, **RobustZScore** (from +Apache Beam's `apache_beam.ml.anomaly` module), and **Threshold** (a simple +fixed-threshold alerter based on a boolean expression). + +## Parameters + +### Required parameters + +* **table**: BigQuery table to monitor. Format: `project:dataset.table`. +* **metric_spec**: JSON string defining the metric computation (see [Metric Spec Reference](#metric-spec-reference) below). +* **detector_spec**: JSON string defining the anomaly detector (see [Detector Spec Reference](#detector-spec-reference) below). +* **topic**: Pub/Sub topic for anomaly results. Full path: `projects//topics/`. + +### Optional parameters +* **poll_interval_sec**: Seconds between BigQuery CDC polls. Default: `60`. +* **change_function**: BigQuery change function: `APPENDS` or `CHANGES`. Default: `APPENDS`. +* **buffer_sec**: Safety buffer behind `now()` in seconds. Default: `15`. +* **start_offset_sec**: Start reading from this many seconds ago. Default: `60`. +* **duration_sec**: How long to run in seconds. `0` means run forever. Default: `0`. +* **temp_dataset**: BigQuery dataset for temp tables. If unset, auto-created. +* **log_all_results**: Log all anomaly detection results (normal, outlier, warmup) at WARNING level. Default: `false`. +* **sink_table**: BigQuery table to write all anomaly detection results to. Format: `project:dataset.table`. If unset, results are not written to BigQuery. +* **decompress_shards**: Number of shards for CDC Arrow batch decompression fan-out. Spreads decompression CPU across workers. `0` disables fan-out (decode inline). Default: `400`. +* **fanout_strategy**: Parallelism strategy for global (non-keyed) metric aggregation: `sharded`, `hotkey_fanout`, or `none`. Ignored when `group_by` is set. Default: `sharded`. See [Fanout Strategies](#fanout-strategies). +* **fanout**: Number of shards for `sharded` or `hotkey_fanout` strategies. Ignored for `none`. Default: `400`. + +## Metric Spec Reference + +The `metric_spec` parameter is a JSON string that defines how raw rows are +aggregated into a single numeric value for anomaly detection. + +```json +{ + "aggregation": { + "window": { + "type": "fixed", + "size_seconds": 3600 + }, + "group_by": ["field1", "field2"], + "measures": [ + {"field": "amount", "agg": "SUM", "alias": "total"} + ] + }, + "derived_fields": [ + {"name": "is_success", "expression": "1 if status == 'success' else 0"} + ], + "measure_combiner": {"expression": "clicks / impressions"} +} +``` + +| Field | Required | Description | +|---|---|---| +| `aggregation` | Yes | Windowed aggregation configuration. | +| `aggregation.window.type` | Yes | `fixed` or `sliding`. | +| `aggregation.window.size_seconds` | Yes | Window size in seconds. | +| `aggregation.window.period_seconds` | Sliding only | Slide period in seconds. | +| `aggregation.group_by` | No | Field names for grouping. Omit for global aggregation. | +| `aggregation.measures` | Yes | List of aggregation measures. | +| `aggregation.measures[].field` | Yes | Input field name (ignored for `COUNT`). | +| `aggregation.measures[].agg` | Yes | `SUM`, `COUNT`, `MIN`, `MAX`, or `MEAN`. | +| `aggregation.measures[].alias` | Yes | Output name for this measure. | +| `derived_fields` | No | Pre-aggregation computed columns. | +| `measure_combiner` | When >1 measure | Post-aggregation expression combining measure aliases. | + +Expressions support: `+`, `-`, `*`, `/`, `//`, `%`, `**`, comparisons, +`and/or/not`, `if/else`, safe builtins (`abs`, `min`, `max`, `round`), +and parentheses. Bare names are field references. + +## Detector Spec Reference + +```json +{"type": "ZScore"} +{"type": "ZScore", "config": {"window_size": 500}} +{"type": "ZScore", "config": {"threshold_criterion": {"type": "FixedThreshold", "config": {"cutoff": 10}}}} +``` + +| Detector | Description | Default threshold | +|---|---|---| +| `ZScore` | `\|value - mean\| / stdev` | 3 | +| `IQR` | Interquartile Range | 1.5 | +| `RobustZScore` | Modified Z-Score using median/MAD | 3.5 | +| `Threshold` | Fixed threshold alert via boolean expression | N/A | + +**Threshold** evaluates a boolean expression against the metric `value` and +fires an alert (label=1) when the expression is true: + +```json +{"type": "Threshold", "expression": "value >= 100"} +{"type": "Threshold", "expression": "value > 100 or value < -100"} +{"type": "Threshold", "expression": "value <= 0.01"} +``` + +The `window_size` shorthand (default: 1000) sets the history buffer for all +internal statistical trackers. + +### Threshold overrides + +```json +{"type": "FixedThreshold", "config": {"cutoff": 10}} +{"type": "QuantileThreshold", "config": {"quantile": 0.95}} +``` + +## Pub/Sub Output + +Detected anomalies (label == 1) are published to the configured Pub/Sub topic +as JSON messages: + +```json +{ + "event_description": "Anomaly detected value=1234.56 score=4.2 in window=2026-03-19T12:00:00.000000Z-2026-03-19T13:00:00.000000Z", + "agent_id": "ZScore", + "key": "(campaign_a, chrome)" +} +``` + +The `key` field is only present for grouped (keyed) metrics. + +Set `--log_all_results` to log all results (normal, outlier, warmup) at +WARNING level in the Dataflow worker logs. + +## Fanout Strategies + +For global aggregation (no `group_by`), all elements are combined under a +single key. At high throughput this creates a bottleneck on the single reducer's +streaming state I/O. The `fanout_strategy` pipeline parameter controls how +elements are distributed across intermediate reducers before the final merge. + +| Strategy | How it works | Best for | +|---|---|---| +| `sharded` (default) | Per-element random sharding into N shard keys. Stage 1 `CombinePerKey` reduces each shard independently. Stage 2 `CombineGlobally` merges N partial accumulators. | High-throughput global aggregation (e.g., 1M+ rows/sec). Uniform distribution regardless of bundle count. | +| `hotkey_fanout` | Beam's built-in `CombineGlobally.with_fanout(N)`. Per-bundle nonce sharding — all elements in one bundle go to the same shard. Better mapper-side pre-combine (PGBK) table efficiency. | When upstream provides many small bundles, or for moderate throughput where PGBK efficiency matters. | +| `none` | Plain `CombineGlobally` with no fanout. Relies on Dataflow's combiner lifting (PGBK) for mapper-side pre-combining. | Low throughput, or when upstream already provides enough parallel bundles (e.g., `decompress_shards`) and streaming state I/O is not a bottleneck. | + +## Getting Started + +### Requirements + +* Java 17 +* Maven 3.9.9+ +* Python 3.11+ +* [gcloud CLI](https://cloud.google.com/sdk/gcloud), and execution of the + following commands: + * `gcloud auth login` + * `gcloud auth application-default login` + +### Required IAM Permissions + +The **worker service account** (used by Dataflow workers) needs: + +| Role | Reason | +|---|---| +| `roles/storage.objectAdmin` | Read/write GCS for staging artifacts and temp files (see note below) | +| `roles/dataflow.developer` | Create and manage Dataflow jobs | +| `roles/bigquery.dataOwner` | Create/delete temp datasets and tables, read CDC data | +| `roles/bigquery.jobUser` | Run BigQuery query jobs | +| `roles/pubsub.editor` | Publish anomaly alerts and verify topic exists | + +The **user or CI account** that launches the template also needs +`roles/iam.serviceAccountUser` on the worker service account to impersonate it. + +> **Note:** If you pre-create the temp dataset with `--temp_dataset`, you can +> scope `roles/bigquery.dataOwner` to just the source and temp datasets +> instead of project-wide, and use `roles/bigquery.dataEditor` if dataset +> deletion is not needed. + +> **Note:** Dataflow auto-creates a default staging bucket +> (`dataflow-staging-{region}-{project_number}`) on first use in a region. +> If this bucket does not exist, the service account needs +> `roles/storage.admin` (or the bucket must be pre-created). Once the +> bucket exists, `roles/storage.objectAdmin` is sufficient. + +### Building the Plugins + +The Maven plugins must be installed before staging: + +```shell +mvn install -pl plugins/core-plugin,plugins/templates-maven-plugin -am -DskipTests -q +``` + +### Staging the Template + +```shell +export PROJECT= +export BUCKET_NAME= + +mvn clean package -PtemplatesStage \ + -DskipTests \ + -DprojectId="$PROJECT" \ + -DbucketName="$BUCKET_NAME" \ + -DstagePrefix="templates" \ + -DtemplateName="BigQuery_Anomaly_Detection" \ + -pl python +``` + +This builds the Docker image, pushes it to `gcr.io/$PROJECT/bigquery-anomaly-detection`, +and writes the template spec to `gs://$BUCKET_NAME/templates/flex/BigQuery_Anomaly_Detection`. + +### Running the Template + +```shell +export PROJECT= +export BUCKET_NAME= +export REGION=us-central1 + +gcloud dataflow flex-template run "bq-anomaly-$(date +%Y%m%d-%H%M%S)" \ + --project "$PROJECT" \ + --region "$REGION" \ + --template-file-gcs-location "gs://$BUCKET_NAME/templates/flex/BigQuery_Anomaly_Detection" \ + --parameters table="$PROJECT:my_dataset.my_table" \ + --parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":60},"measures":[{"field":"amount","agg":"SUM","alias":"revenue"}]}}' \ + --parameters detector_spec='{"type":"ZScore"}' \ + --parameters topic="projects/$PROJECT/topics/bqmonitor-anomalies" \ + --parameters duration_sec="300" +``` + +Or run directly from the container image (skipping the GCS spec file): + +```shell +gcloud dataflow flex-template run "bq-anomaly-test" \ + --image "gcr.io/$PROJECT/bigquery-anomaly-detection:templates" \ + --project "$PROJECT" \ + --region "$REGION" \ + --sdk-language PYTHON \ + --parameters table="$PROJECT:my_dataset.my_table" \ + --parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":60},"measures":[{"field":"amount","agg":"SUM","alias":"revenue"}]}}' \ + --parameters detector_spec='{"type":"ZScore"}' +``` + +### Regenerating Pinned Dependencies + +The `requirements.txt` contains pinned and hashed dependencies. To regenerate +after changing base dependencies: + +```shell +# Edit python/default_base_bqmonitor_requirements.txt, then: +sh python/generate_all_dependencies.sh +``` + +### Running Integration Tests + +The integration tests stage the template, launch it against real BigQuery and +Pub/Sub resources, and verify end-to-end anomaly detection. + +```shell +export PROJECT= +export REGION=us-east5 +export BUCKET_NAME= + +# Build plugins first (one-time). +mvn install -pl plugins/core-plugin,plugins/templates-maven-plugin -am -DskipTests -q + +# Run the integration tests. +mvn verify -PtemplatesIntegrationTests \ + -Dproject="$PROJECT" \ + -Dregion="$REGION" \ + -DartifactBucket="gs://$BUCKET_NAME" \ + -pl python \ + -Dtest=BigQueryAnomalyDetectionIT +``` + +To run a single test method: + +```shell +mvn verify -PtemplatesIntegrationTests \ + -Dproject="$PROJECT" \ + -Dregion="$REGION" \ + -DartifactBucket="gs://$BUCKET_NAME" \ + -pl python \ + -Dtest=BigQueryAnomalyDetectionIT#testDetectsAnomalyAndPublishesToPubSub +``` + +The test service account needs all roles listed in +[Required IAM Permissions](#required-iam-permissions) plus the ability to +create and delete test resources (Pub/Sub topics/subscriptions, BigQuery +datasets/tables). + +## Examples + +### Simple SUM metric + +```shell +--parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' +--parameters detector_spec='{"type":"ZScore"}' +``` + +### Grouped ratio metric (CTR) + +```shell +--parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":60},"group_by":["campaign_type","browser"],"measures":[{"field":"is_click","agg":"SUM","alias":"clicks"},{"field":"is_click","agg":"COUNT","alias":"impressions"}]},"measure_combiner":{"expression":"clicks / impressions"}}' +--parameters detector_spec='{"type":"ZScore"}' +``` + +### Derived field + custom threshold + +```shell +--parameters metric_spec='{"derived_fields":[{"name":"is_success","expression":"1 if status == '"'"'success'"'"' else 0"}],"aggregation":{"window":{"type":"fixed","size_seconds":60},"group_by":["brand"],"measures":[{"field":"is_success","agg":"SUM","alias":"successes"},{"field":"is_success","agg":"COUNT","alias":"total"}]},"measure_combiner":{"expression":"successes / total"}}' +--parameters detector_spec='{"type":"ZScore","config":{"threshold_criterion":{"type":"FixedThreshold","config":{"cutoff":10}}}}' +``` + +## Project Structure + +``` +python/src/main/python/bigquery-anomaly-detection/ + main.py # Entry point + setup.py # Package configuration + pyproject.toml # Build system config + requirements.txt # Pinned dependencies (generated) + src/bqmonitor/ + __init__.py + pipeline.py # Pipeline construction and options + cdc.py # BigQuery CDC reader (ReadBigQueryChangeHistory) + metric.py # MetricSpec and ComputeMetric PTransform + safe_eval.py # Safe expression evaluation (Expr) + +python/src/main/java/.../BigQueryAnomalyDetection.java # Template metadata +python/src/test/java/.../BigQueryAnomalyDetectionIT.java # Integration test +python/default_base_bqmonitor_requirements.txt # Base dependencies +``` diff --git a/python/default_base_bqmonitor_requirements.txt b/python/default_base_bqmonitor_requirements.txt new file mode 100644 index 0000000000..abd9ea8999 --- /dev/null +++ b/python/default_base_bqmonitor_requirements.txt @@ -0,0 +1,3 @@ +apache-beam[gcp]==2.71.0 +google-cloud-bigquery-storage +setuptools diff --git a/python/generate_all_dependencies.sh b/python/generate_all_dependencies.sh index 1e0035cdd5..d02635972a 100755 --- a/python/generate_all_dependencies.sh +++ b/python/generate_all_dependencies.sh @@ -20,6 +20,7 @@ set -e SCRIPTPATH=$(dirname "$0") sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/streaming-llm/base_requirements.txt $SCRIPTPATH/../python/src/main/python/streaming-llm/requirements.txt +sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_bqmonitor_requirements.txt $SCRIPTPATH/../python/src/main/python/bigquery-anomaly-detection/requirements_all.txt # Generate a base set of dependencies to use for any templates without special dependencies mkdir -p $SCRIPTPATH/__build__/ sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/default_base_python_requirements.txt $SCRIPTPATH/__build__/default_python_requirements.txt diff --git a/python/pom.xml b/python/pom.xml index af8a7bf57c..56bff10c5f 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -83,6 +83,74 @@ + + bqmonitorPythonTests + + false + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + bqmonitor-pip-install + test-compile + + exec + + + pip + + install + -r + src/test/python/bigquery-anomaly-detection/requirements-test.txt + + + + + bqmonitor-pip-install-pkg + test-compile + + exec + + + pip + + install + -e + src/main/python/bigquery-anomaly-detection + + + + + bqmonitor-python-test + test + + exec + + + python + ${project.basedir} + + -m + unittest + discover + -s + src/test/python/bigquery-anomaly-detection + -p + *_test.py + -v + + + + + + + + templatesValidate diff --git a/python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java b/python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java new file mode 100644 index 0000000000..5d8fbadc6a --- /dev/null +++ b/python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.templates.python; + +import com.google.cloud.teleport.metadata.Template; +import com.google.cloud.teleport.metadata.Template.TemplateType; +import com.google.cloud.teleport.metadata.TemplateCategory; +import com.google.cloud.teleport.metadata.TemplateParameter; + +/** Template class for BigQuery Anomaly Detection in Python. */ +@Template( + name = "BigQuery_Anomaly_Detection", + category = TemplateCategory.STREAMING, + type = TemplateType.PYTHON, + displayName = "BigQuery Anomaly Detection", + description = + "[Experimental] Real-time anomaly detection on BigQuery change data (CDC). " + + "Reads streaming APPENDS/CHANGES data from a BigQuery table, " + + "computes a configurable windowed metric, runs anomaly detection " + + "(ZScore, IQR, or RobustZScore), and publishes anomalies to Pub/Sub.", + preview = true, + flexContainerName = "bigquery-anomaly-detection", + filesToCopy = {"main.py", "setup.py", "pyproject.toml", "requirements_all.txt", "src"}, + contactInformation = "https://cloud.google.com/support", + streaming = true) +public interface BigQueryAnomalyDetection { + + @TemplateParameter.Text( + order = 1, + name = "table", + description = "BigQuery Table", + helpText = "BigQuery table to monitor. Format: project:dataset.table", + regexes = {"^[a-zA-Z0-9_-]+:[a-zA-Z0-9_]+\\.[a-zA-Z0-9_]+$"}) + String getTable(); + + @TemplateParameter.Text( + order = 2, + name = "metric_spec", + description = "Metric Specification (JSON)", + helpText = + "JSON string defining the metric computation. " + + "Example: {\"aggregation\":{\"window\":{\"type\":\"fixed\"," + + "\"size_seconds\":3600},\"measures\":[{\"field\":\"amount\",\"agg\":\"SUM\"," + + "\"alias\":\"total\"}]}}") + String getMetricSpec(); + + @TemplateParameter.Text( + order = 3, + name = "detector_spec", + description = "Detector Specification (JSON)", + helpText = + "JSON string defining the anomaly detector. " + + "Example: {\"type\":\"ZScore\"} or " + + "{\"type\":\"ZScore\",\"config\":{\"threshold_criterion\":{\"type\":\"FixedThreshold\"," + + "\"config\":{\"cutoff\":10}}}}") + String getDetectorSpec(); + + @TemplateParameter.Integer( + order = 5, + optional = true, + name = "poll_interval_sec", + description = "Poll Interval (seconds)", + helpText = "Seconds between BigQuery CDC polls. Default: 60.") + Integer getPollIntervalSec(); + + @TemplateParameter.Text( + order = 6, + optional = true, + name = "change_function", + description = "Change Function", + helpText = "BigQuery change function: APPENDS or CHANGES. Default: APPENDS.", + regexes = {"^(APPENDS|CHANGES)$"}) + String getChangeFunction(); + + @TemplateParameter.Integer( + order = 7, + optional = true, + name = "buffer_sec", + description = "Buffer (seconds)", + helpText = "Safety buffer behind now() in seconds. Default: 15.") + Integer getBufferSec(); + + @TemplateParameter.Integer( + order = 8, + optional = true, + name = "start_offset_sec", + description = "Start Offset (seconds)", + helpText = "Start reading from this many seconds ago. Default: 60.") + Integer getStartOffsetSec(); + + @TemplateParameter.Integer( + order = 9, + optional = true, + name = "duration_sec", + description = "Duration (seconds)", + helpText = "How long to run in seconds. 0 means run forever. Default: 0.") + Integer getDurationSec(); + + @TemplateParameter.Text( + order = 10, + optional = true, + name = "temp_dataset", + description = "Temp Dataset", + helpText = "BigQuery dataset for temp tables. If unset, auto-created.") + String getTempDataset(); + + @TemplateParameter.Text( + order = 4, + name = "topic", + description = "Pub/Sub Topic", + helpText = + "Pub/Sub topic for anomaly results. " + "Full path: projects//topics/.") + String getTopic(); + + @TemplateParameter.Boolean( + order = 11, + optional = true, + name = "log_all_results", + description = "Log All Results", + helpText = + "Log all anomaly detection results (normal, outlier, warmup) " + + "at WARNING level. Default: false.") + Boolean getLogAllResults(); + + @TemplateParameter.Text( + order = 12, + optional = true, + name = "sink_table", + description = "Sink BigQuery Table", + helpText = + "BigQuery table to write all anomaly detection results to. " + + "Format: project:dataset.table. If unset, results are not written to BigQuery.", + regexes = {"^[a-zA-Z0-9_-]+:[a-zA-Z0-9_]+\\.[a-zA-Z0-9_]+$"}) + String getSinkTable(); + + @TemplateParameter.Integer( + order = 13, + optional = true, + name = "decompress_shards", + description = "Decompress Shards", + helpText = + "Number of shards for CDC Arrow batch decompression fan-out. " + + "Spreads decompression CPU across workers. " + + "0 disables fan-out (decode inline). Default: 400.") + Integer getDecompressShards(); + + @TemplateParameter.Text( + order = 14, + optional = true, + name = "fanout_strategy", + description = "Fanout Strategy", + helpText = + "Parallelism strategy for global (non-keyed) metric aggregation: " + + "sharded, hotkey_fanout, or none. " + + "Ignored when group_by is set. Default: sharded.", + regexes = {"^(sharded|hotkey_fanout|none)$"}) + String getFanoutStrategy(); + + @TemplateParameter.Integer( + order = 15, + optional = true, + name = "fanout", + description = "Fanout Shards", + helpText = + "Number of shards for sharded or hotkey_fanout strategies. " + + "Ignored for none. Default: 400.") + Integer getFanout(); + + @TemplateParameter.Boolean( + order = 16, + optional = true, + name = "mapper_side_precombine", + description = "Mapper-Side Pre-Combine", + helpText = + "Enable mapper-side pre-aggregation within each bundle before shuffle. " + + "Reduces shuffle volume by folding values per key locally (like " + + "PGBKCVOperation) while preserving Dataflow's incremental state-side " + + "merging on the downstream CombinePerKey. Effective when few keys " + + "see high throughput (>100k rows/sec/key). Default: false.") + Boolean getMapperSidePrecombine(); +} diff --git a/python/src/main/python/bigquery-anomaly-detection/main.py b/python/src/main/python/bigquery-anomaly-detection/main.py new file mode 100644 index 0000000000..afd7bf0281 --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/main.py @@ -0,0 +1,25 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Flex Template entry point for bqmonitor.""" + +import logging + +from bqmonitor.pipeline import run + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/python/src/main/python/bigquery-anomaly-detection/pyproject.toml b/python/src/main/python/bigquery-anomaly-detection/pyproject.toml new file mode 100644 index 0000000000..c36dd5d36d --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "bqmonitor" +version = "0.1.0" +description = "BigQuery anomaly monitoring pipeline (Dataflow Flex Template)" +requires-python = ">=3.11" +dependencies = [ + "apache-beam[gcp]==2.71.0", + "google-cloud-bigquery-storage", +] diff --git a/python/src/main/python/bigquery-anomaly-detection/requirements_all.txt b/python/src/main/python/bigquery-anomaly-detection/requirements_all.txt new file mode 100644 index 0000000000..ecae6a5363 --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/requirements_all.txt @@ -0,0 +1,2579 @@ +# Copyright 2025 Google Inc. All Rights Reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated requirements file for Apache Beam container image. +# From the templates base directory to update, +# run: sh python/generate_all_dependencies.sh +# Do not edit manually, adjust the base requirements file, and regenerate the list. + +# See [maintainers-guide](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/contributor-docs/maintainers-guide.md#validating-and-upgrading-beam-versions) for more information. + +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --output-file=python/../python/src/main/python/bigquery-anomaly-detection/requirements.txt python/default_base_bqmonitor_requirements.txt +# +aiofiles==25.1.0 \ + --hash=sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2 \ + --hash=sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695 + # via cloud-sql-python-connector +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 + # via aiohttp +aiohttp==3.13.3 \ + --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf \ + --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c \ + --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c \ + --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 \ + --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f \ + --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 \ + --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 \ + --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf \ + --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 \ + --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 \ + --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 \ + --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 \ + --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d \ + --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea \ + --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 \ + --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 \ + --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 \ + --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 \ + --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 \ + --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 \ + --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a \ + --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e \ + --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 \ + --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 \ + --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 \ + --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd \ + --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a \ + --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 \ + --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 \ + --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 \ + --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f \ + --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 \ + --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 \ + --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 \ + --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 \ + --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce \ + --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 \ + --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 \ + --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 \ + --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 \ + --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b \ + --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 \ + --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c \ + --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 \ + --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 \ + --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f \ + --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b \ + --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e \ + --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 \ + --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 \ + --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 \ + --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d \ + --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 \ + --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 \ + --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce \ + --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 \ + --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 \ + --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c \ + --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf \ + --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 \ + --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 \ + --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e \ + --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 \ + --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a \ + --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 \ + --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a \ + --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 \ + --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 \ + --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba \ + --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df \ + --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 \ + --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 \ + --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 \ + --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 \ + --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc \ + --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 \ + --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 \ + --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 \ + --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 \ + --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 \ + --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 \ + --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 \ + --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 \ + --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 \ + --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 \ + --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc \ + --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 \ + --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 \ + --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 \ + --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 \ + --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 \ + --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 \ + --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c \ + --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 \ + --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 \ + --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a \ + --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 \ + --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 \ + --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 \ + --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d \ + --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 \ + --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d \ + --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f \ + --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f \ + --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 \ + --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 \ + --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 \ + --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 \ + --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af \ + --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 \ + --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 \ + --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 \ + --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c \ + --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 \ + --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 \ + --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 \ + --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 \ + --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 \ + --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa \ + --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 + # via cloud-sql-python-connector +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp +annotated-types==0.7.0 \ + --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ + --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 + # via pydantic +anyio==4.12.1 \ + --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ + --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c + # via + # google-genai + # httpx +apache-beam[gcp]==2.71.0 \ + --hash=sha256:044841032ef190a7ad69a9d4ca4b23c104a310d08c47a1f5faefcf830c9e5520 \ + --hash=sha256:192b00d13de8eb06241c5332ecef7a9a947758e2103b07d6726848ba9f0b5a49 \ + --hash=sha256:1cdaf7e502da67f674ecf8dd8cec21252bd1b2678a5d18b873a45635cf0e7cec \ + --hash=sha256:28c6eeb05b688dcc503fce84075fcd03a73bbd9e449e70521f2efb47a932bcea \ + --hash=sha256:317f5495c3266b9146263dbb881110b56b015fbc7e2f1e27eb9932b2bf28a94c \ + --hash=sha256:3705d824d462aee4bf162318eb0ef1ca767064e73aa4f1ba14d741cc12c19143 \ + --hash=sha256:43ed7ae3dbecf67af2ad412b86d160fc6177d19fc6e59ed18aee4a84355858db \ + --hash=sha256:515064493c478e92a87618f46c8b8c2143ce244317db683dc3d824fda37b0db5 \ + --hash=sha256:5ca7fca47ae39b5e6497c39bca303d11c200fdfae6b352e5e481a59a9b886f75 \ + --hash=sha256:78c2f8e88014555984a7a21bcb63479e135b958428d178d45699a4154ae84634 \ + --hash=sha256:78e3e913275bd1c1aac1ecc90af78fb65915908671b6e39d60a3a31de3438782 \ + --hash=sha256:81766907e53a5feddb2d1b5553c6f1154ff7cae67e548b4c2726e299334572bf \ + --hash=sha256:8189d2e1d314a7dc8f3456bae4c7641637d302490e1af93db3aa6ba45d716b70 \ + --hash=sha256:83be2fce3726529f221c8d99f844f64d68494b2bad438852f96f02f2c0e8cac8 \ + --hash=sha256:8e1cbc386cf8c0d740b3b2847cb7c99481672ed036b57c11eb2f41d049800b40 \ + --hash=sha256:a11147b82260d69b19021b32a65da044d38f65195ec2a66460ccad80649106b5 \ + --hash=sha256:a14fb6972de7113dfbe6bba967de1a3a5c60228a96b96eb32a675762f83d659b \ + --hash=sha256:a358a7e689e1acb903ec5f545ed22b674fb6cbb17424518630412cba3a627937 \ + --hash=sha256:a7967a1d75daec31e9d03705304ad4e7e5bcad266dd5e8bad98a68e76ebb368f \ + --hash=sha256:af5a9acf850b8430440f8e6f687650c252dd7d0b929fbef2d84ce79087f6bb6b \ + --hash=sha256:b3acb72a5afdc15abe696e37915cbce91d7a0672fda2658c2185d8ea4684d4e3 \ + --hash=sha256:c015aa7ee75cabc58277b19317429fc3ed08752173d6750b2212260190505c7f \ + --hash=sha256:d4a3b4008ca3966f426a8580535e2227387518a2d62c3928c4e3d5a6ca23dd8a \ + --hash=sha256:de890d820ae365eddcbe522e61816a967ab9d5be501fb56435e0d8a8c571408e \ + --hash=sha256:e06fb7fd4f5aa9d16bb8d8d30d9c24fc255cfc9be510188bfab0b11f398cc515 + # via -r python/default_base_bqmonitor_requirements.txt +asn1crypto==1.5.1 \ + --hash=sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c \ + --hash=sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67 + # via scramp +attrs==25.4.0 \ + --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ + --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 + # via aiohttp +backports-tarfile==1.2.0 \ + --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ + --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 + # via jaraco-context +beartype==0.22.9 \ + --hash=sha256:8f82b54aa723a2848a56008d18875f91c1db02c32ef6a62319a002e3e25a975f \ + --hash=sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2 + # via apache-beam +betterproto==2.0.0b7 \ + --hash=sha256:1b1458ca5278d519bcd62556a4c236f998a91d503f0f71c67b0b954747052af2 \ + --hash=sha256:401ab8055e2f814e77b9c88a74d0e1ae3d1e8a969cced6aeb1b59f71ad63fbd2 + # via envoy-data-plane +cachetools==6.2.6 \ + --hash=sha256:16c33e1f276b9a9c0b49ab5782d901e3ad3de0dd6da9bf9bcd29ac5672f2f9e6 \ + --hash=sha256:8c9717235b3c651603fff0076db52d6acbfd1b338b8ed50256092f7ce9c85bda + # via apache-beam +certifi==2026.2.25 \ + --hash=sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa \ + --hash=sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7 + # via + # httpcore + # httpx + # requests +cffi==2.0.0 \ + --hash=sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb \ + --hash=sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b \ + --hash=sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f \ + --hash=sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9 \ + --hash=sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44 \ + --hash=sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2 \ + --hash=sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c \ + --hash=sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75 \ + --hash=sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65 \ + --hash=sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e \ + --hash=sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a \ + --hash=sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e \ + --hash=sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25 \ + --hash=sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a \ + --hash=sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe \ + --hash=sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b \ + --hash=sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91 \ + --hash=sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592 \ + --hash=sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187 \ + --hash=sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c \ + --hash=sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1 \ + --hash=sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94 \ + --hash=sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba \ + --hash=sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb \ + --hash=sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165 \ + --hash=sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529 \ + --hash=sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca \ + --hash=sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c \ + --hash=sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6 \ + --hash=sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c \ + --hash=sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0 \ + --hash=sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743 \ + --hash=sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63 \ + --hash=sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5 \ + --hash=sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5 \ + --hash=sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4 \ + --hash=sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d \ + --hash=sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b \ + --hash=sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93 \ + --hash=sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205 \ + --hash=sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27 \ + --hash=sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512 \ + --hash=sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d \ + --hash=sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c \ + --hash=sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037 \ + --hash=sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26 \ + --hash=sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322 \ + --hash=sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb \ + --hash=sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c \ + --hash=sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8 \ + --hash=sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4 \ + --hash=sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414 \ + --hash=sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9 \ + --hash=sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664 \ + --hash=sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9 \ + --hash=sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775 \ + --hash=sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739 \ + --hash=sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc \ + --hash=sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062 \ + --hash=sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe \ + --hash=sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9 \ + --hash=sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92 \ + --hash=sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5 \ + --hash=sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13 \ + --hash=sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d \ + --hash=sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26 \ + --hash=sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f \ + --hash=sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495 \ + --hash=sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b \ + --hash=sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6 \ + --hash=sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c \ + --hash=sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef \ + --hash=sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5 \ + --hash=sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18 \ + --hash=sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad \ + --hash=sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3 \ + --hash=sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7 \ + --hash=sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5 \ + --hash=sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534 \ + --hash=sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49 \ + --hash=sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2 \ + --hash=sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5 \ + --hash=sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453 \ + --hash=sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf + # via cryptography +charset-normalizer==3.4.5 \ + --hash=sha256:014837af6fabf57121b6254fa8ade10dceabc3528b27b721a64bbc7b8b1d4eb4 \ + --hash=sha256:01a1ed54b953303ca7e310fafe0fe347aab348bd81834a0bcd602eb538f89d66 \ + --hash=sha256:0294916d6ccf2d069727d65973c3a1ca477d68708db25fd758dd28b0827cff54 \ + --hash=sha256:02a9d1b01c1e12c27883b0c9349e0bcd9ae92e727ff1a277207e1a262b1cbf05 \ + --hash=sha256:036c079aa08a6a592b82487f97c60b439428320ed1b2ea0b3912e99d30c77765 \ + --hash=sha256:039215608ac7b358c4da0191d10fc76868567fbf276d54c14721bdedeb6de064 \ + --hash=sha256:0625665e4ebdddb553ab185de5db7054393af8879fb0c87bd5690d14379d6819 \ + --hash=sha256:0a45e504f5e1be0bd385935a8e1507c442349ca36f511a47057a71c9d1d6ea9e \ + --hash=sha256:0b362bcd27819f9c07cbf23db4e0e8cd4b44c5ecd900c2ff907b2b92274a7412 \ + --hash=sha256:0c300cefd9b0970381a46394902cd18eaf2aa00163f999590ace991989dcd0fc \ + --hash=sha256:1088345bcc93c58d8d8f3d783eca4a6e7a7752bbff26c3eee7e73c597c191c2e \ + --hash=sha256:10b473fc8dca1c3ad8559985794815f06ca3fc71942c969129070f2c3cdf7281 \ + --hash=sha256:131716d6786ad5e3dc542f5cc6f397ba3339dc0fb87f87ac30e550e8987756af \ + --hash=sha256:14498a429321de554b140013142abe7608f9d8ccc04d7baf2ad60498374aefa2 \ + --hash=sha256:149ec69866c3d6c2fb6f758dbc014ecb09f30b35a5ca90b6a8a2d4e54e18fdfe \ + --hash=sha256:165c7b21d19365464e8f70e5ce5e12524c58b48c78c1f5a57524603c1ab003f8 \ + --hash=sha256:1827734a5b308b65ac54e86a618de66f935a4f63a8a462ff1e19a6788d6c2262 \ + --hash=sha256:19092dde50335accf365cce21998a1c6dd8eafd42c7b226eb54b2747cdce2fac \ + --hash=sha256:1a374cc0b88aa710e8865dc1bd6edb3743c59f27830f0293ab101e4cf3ce9f85 \ + --hash=sha256:1d1401945cb77787dbd3af2446ff2d75912327c4c3a1526ab7955ecf8600687c \ + --hash=sha256:1f2da5cbb9becfcd607757a169e38fb82aa5fd86fae6653dea716e7b613fe2cf \ + --hash=sha256:259cd1ca995ad525f638e131dbcc2353a586564c038fc548a3fe450a91882139 \ + --hash=sha256:2820a98460c83663dd8ec015d9ddfd1e4879f12e06bb7d0500f044fb477d2770 \ + --hash=sha256:28269983f25a4da0425743d0d257a2d6921ea7d9b83599d4039486ec5b9f911d \ + --hash=sha256:2b970382e4a36bed897c19f310f31d7d13489c11b4f468ddfba42d41cddfb918 \ + --hash=sha256:2da4eedcb6338e2321e831a0165759c0c620e37f8cd044a263ff67493be8ffb3 \ + --hash=sha256:30987f4a8ed169983f93e1be8ffeea5214a779e27ed0b059835c7afe96550ad7 \ + --hash=sha256:30a2b1a48478c3428d047ed9690d57c23038dac838a87ad624c85c0a78ebeb39 \ + --hash=sha256:340810d34ef83af92148e96e3e44cb2d3f910d2bf95e5618a5c467d9f102231d \ + --hash=sha256:3f64c6bf8f32f9133b668c7f7a7cbdbc453412bc95ecdbd157f3b1e377a92990 \ + --hash=sha256:4167a621a9a1a986c73777dbc15d4b5eac8ac5c10393374109a343d4013ec765 \ + --hash=sha256:4354e401eb6dab9aed3c7b4030514328a6c748d05e1c3e19175008ca7de84fb1 \ + --hash=sha256:4481e6da1830c8a1cc0b746b47f603b653dadb690bcd851d039ffaefe70533aa \ + --hash=sha256:4b8551b6e6531e156db71193771c93bda78ffc4d1e6372517fe58ad3b91e4659 \ + --hash=sha256:4cd966c2559f501c6fd69294d082c2934c8dd4719deb32c22961a5ac6db0df1d \ + --hash=sha256:50bcbca6603c06a1dcc7b056ed45c37715fb5d2768feb3bcd37d2313c587a5b9 \ + --hash=sha256:530beedcec9b6e027e7a4b6ce26eed36678aa39e17da85e6e03d7bd9e8e9d7c9 \ + --hash=sha256:568e3c34b58422075a1b49575a6abc616d9751b4d61b23f712e12ebb78fe47b2 \ + --hash=sha256:573ef5814c4b7c0d59a7710aa920eaaaef383bd71626aa420fba27b5cab92e8d \ + --hash=sha256:58ad8270cfa5d4bef1bc85bd387217e14ff154d6630e976c6f56f9a040757475 \ + --hash=sha256:597d10dec876923e5c59e48dbd366e852eacb2b806029491d307daea6b917d7c \ + --hash=sha256:5bcb3227c3d9aaf73eaaab1db7ccd80a8995c509ee9941e2aae060ca6e4e5d81 \ + --hash=sha256:5cffde4032a197bd3b42fd0b9509ec60fb70918d6970e4cc773f20fc9180ca67 \ + --hash=sha256:5fea359734b140d0d6741189fea5478c6091b54ffc69d7ce119e0a05637d8c99 \ + --hash=sha256:60d68e820af339df4ae8358c7a2e7596badeb61e544438e489035f9fbf3246a5 \ + --hash=sha256:610f72c0ee565dfb8ae1241b666119582fdbfe7c0975c175be719f940e110694 \ + --hash=sha256:65a126fb4b070d05340a84fc709dd9e7c75d9b063b610ece8a60197a291d0adf \ + --hash=sha256:65b3c403a5b6b8034b655e7385de4f72b7b244869a22b32d4030b99a60593eca \ + --hash=sha256:66dee73039277eb35380d1b82cccc69cc82b13a66f9f4a18da32d573acf02b7c \ + --hash=sha256:708c7acde173eedd4bfa4028484426ba689d2103b28588c513b9db2cd5ecde9c \ + --hash=sha256:728c6a963dfab66ef865f49286e45239384249672cd598576765acc2a640a636 \ + --hash=sha256:754f96058e61a5e22e91483f823e07df16416ce76afa4ebf306f8e1d1296d43f \ + --hash=sha256:75dfd1afe0b1647449e852f4fb428195a7ed0588947218f7ba929f6538487f02 \ + --hash=sha256:75ee9c1cce2911581a70a3c0919d8bccf5b1cbc9b0e5171400ec736b4b569497 \ + --hash=sha256:76a9d0de4d0eab387822e7b35d8f89367dd237c72e82ab42b9f7bf5e15ada00f \ + --hash=sha256:77be992288f720306ab4108fe5c74797de327f3248368dfc7e1a916d6ed9e5a2 \ + --hash=sha256:7ad83b8f9379176c841f8865884f3514d905bcd2a9a3b210eaa446e7d2223e4d \ + --hash=sha256:8197abe5ca1ffb7d91e78360f915eef5addff270f8a71c1fc5be24a56f3e4873 \ + --hash=sha256:82cc7c2ad42faec8b574351f8bc2a0c049043893853317bd9bb309f5aba6cb5a \ + --hash=sha256:8a28afb04baa55abf26df544e3e5c6534245d3daa5178bc4a8eeb48202060d0e \ + --hash=sha256:8b78d8a609a4b82c273257ee9d631ded7fac0d875bdcdccc109f3ee8328cfcb1 \ + --hash=sha256:8ce11cd4d62d11166f2b441e30ace226c19a3899a7cf0796f668fba49a9fb123 \ + --hash=sha256:8fff79bf5978c693c9b1a4d71e4a94fddfb5fe744eb062a318e15f4a2f63a550 \ + --hash=sha256:92263f7eca2f4af326cd20de8d16728d2602f7cfea02e790dcde9d83c365d7cc \ + --hash=sha256:93b3b2cc5cf1b8743660ce77a4f45f3f6d1172068207c1defc779a36eea6bb36 \ + --hash=sha256:95adae7b6c42a6c5b5b559b1a99149f090a57128155daeea91732c8d970d8644 \ + --hash=sha256:97ab7787092eb9b50fb47fa04f24c75b768a606af1bcba1957f07f128a7219e4 \ + --hash=sha256:9db5e3fcdcee89a78c04dffb3fe33c79f77bd741a624946db2591c81b2fc85b0 \ + --hash=sha256:a118e2e0b5ae6b0120d5efa5f866e58f2bb826067a646431da4d6a2bdae7950e \ + --hash=sha256:a2aecdb364b8a1802afdc7f9327d55dad5366bc97d8502d0f5854e50712dbc5f \ + --hash=sha256:a66aa5022bf81ab4b1bebfb009db4fd68e0c6d4307a1ce5ef6a26e5878dfc9e4 \ + --hash=sha256:a68766a3c58fde7f9aaa22b3786276f62ab2f594efb02d0a1421b6282e852e98 \ + --hash=sha256:aa2f963b4da26daf46231d9b9e0e2c9408a751f8f0d0f44d2de56d3caf51d294 \ + --hash=sha256:aa92ec1102eaff840ccd1021478af176a831f1bccb08e526ce844b7ddda85c22 \ + --hash=sha256:ac59c15e3f1465f722607800c68713f9fbc2f672b9eb649fe831da4019ae9b23 \ + --hash=sha256:ae8b03427410731469c4033934cf473426faff3e04b69d2dfb64a4281a3719f8 \ + --hash=sha256:afca7f78067dd27c2b848f1b234623d26b87529296c6c5652168cc1954f2f3b2 \ + --hash=sha256:b2d37d78297b39a9eb9eb92c0f6df98c706467282055419df141389b23f93362 \ + --hash=sha256:b3e71afc578b98512bfe7bdb822dd6bc57d4b0093b4b6e5487c1e96ad4ace242 \ + --hash=sha256:ba20bdf69bd127f66d0174d6f2a93e69045e0b4036dc1ca78e091bcc765830c4 \ + --hash=sha256:c108f8619e504140569ee7de3f97d234f0fbae338a7f9f360455071ef9855a95 \ + --hash=sha256:c23eb3263356d94858655b3e63f85ac5d50970c6e8febcdde7830209139cc37d \ + --hash=sha256:c5af897b45fa606b12464ccbe0014bbf8c09191e0a66aab6aa9d5cf6e77e0c94 \ + --hash=sha256:c7a80a9242963416bd81f99349d5f3fce1843c303bd404f204918b6d75a75fd6 \ + --hash=sha256:c7e84e0c0005e3bdc1a9211cd4e62c78ba80bc37b2365ef4410cd2007a9047f2 \ + --hash=sha256:cace89841c0599d736d3d74a27bc5821288bb47c5441923277afc6059d7fbcb4 \ + --hash=sha256:cd2d0f0ec9aa977a27731a3209ebbcacebebaf41f902bd453a928bfd281cf7f8 \ + --hash=sha256:d01de5e768328646e6a3fa9e562706f8f6641708c115c62588aef2b941a4f88e \ + --hash=sha256:d1028de43596a315e2720a9849ee79007ab742c06ad8b45a50db8cdb7ed4a82a \ + --hash=sha256:d27ce22ec453564770d29d03a9506d449efbb9fa13c00842262b2f6801c48cce \ + --hash=sha256:d29dd9c016f2078b43d0c357511e87eee5b05108f3dd603423cb389b89813969 \ + --hash=sha256:d31f0d1671e1534e395f9eb84a68e0fb670e1edb1fe819a9d7f564ae3bc4e53f \ + --hash=sha256:d4eb8ac7469b2a5d64b5b8c04f84d8bf3ad340f4514b98523805cbf46e3b3923 \ + --hash=sha256:d5e52d127045d6ae01a1e821acfad2f3a1866c54d0e837828538fabe8d9d1bd6 \ + --hash=sha256:d77f97e515688bd615c1d1f795d540f32542d514242067adcb8ef532504cb9ee \ + --hash=sha256:d8ed79b8f6372ca4254955005830fd61c1ccdd8c0fac6603e2c145c61dd95db6 \ + --hash=sha256:dc57a0baa3eeedd99fafaef7511b5a6ef4581494e8168ee086031744e2679467 \ + --hash=sha256:e09f671a54ce70b79a1fc1dc6da3072b7ef7251fadb894ed92d9aa8218465a5f \ + --hash=sha256:e22d1059b951e7ae7c20ef6b06afd10fb95e3c41bf3c4fbc874dba113321c193 \ + --hash=sha256:e37bd100d2c5d3ba35db9c7c5ba5a9228cbcffe5c4778dc824b164e5257813d7 \ + --hash=sha256:e51ae7d81c825761d941962450f50d041db028b7278e7b08930b4541b3e45cb9 \ + --hash=sha256:e545b51da9f9af5c67815ca0eb40676c0f016d0b0381c86f20451e35696c5f95 \ + --hash=sha256:e6302ca4ae283deb0af68d2fbf467474b8b6aedcd3dab4db187e07f94c109763 \ + --hash=sha256:e71bbb595973622b817c042bd943c3f3667e9c9983ce3d205f973f486fec98a7 \ + --hash=sha256:ec56a2266f32bc06ed3c3e2a8f58417ce02f7e0356edc89786e52db13c593c98 \ + --hash=sha256:ed1a9a204f317ef879b32f9af507d47e49cd5e7f8e8d5d96358c98373314fc60 \ + --hash=sha256:ed97c282ee4f994ef814042423a529df9497e3c666dca19be1d4cd1129dc7ade \ + --hash=sha256:ed98364e1c262cf5f9363c3eca8c2df37024f52a8fa1180a3610014f26eac51c \ + --hash=sha256:ee57b926940ba00bca7ba7041e665cc956e55ef482f851b9b65acb20d867e7a2 \ + --hash=sha256:f1d725b754e967e648046f00c4facc42d414840f5ccc670c5670f59f83693e4f \ + --hash=sha256:f8102ae93c0bc863b1d41ea0f4499c20a83229f52ed870850892df555187154a \ + --hash=sha256:fc1c64934b8faf7584924143eb9db4770bbdb16659626e1a1a4d9efbcb68d947 \ + --hash=sha256:ff95a9283de8a457e6b12989de3f9f5193430f375d64297d323a615ea52cbdb3 + # via requests +cloud-sql-python-connector==1.20.0 \ + --hash=sha256:aa7c30631c5f455d14d561d7b0b414a97652a1b582a301f5570ba2cea2aa9105 \ + --hash=sha256:fdd96153b950040b0252453115604c142922b72cf3636146165a648ac5f6fc30 + # via apache-beam +cryptography==46.0.5 \ + --hash=sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72 \ + --hash=sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235 \ + --hash=sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9 \ + --hash=sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356 \ + --hash=sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257 \ + --hash=sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad \ + --hash=sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4 \ + --hash=sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c \ + --hash=sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614 \ + --hash=sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed \ + --hash=sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31 \ + --hash=sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229 \ + --hash=sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0 \ + --hash=sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731 \ + --hash=sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b \ + --hash=sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4 \ + --hash=sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4 \ + --hash=sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263 \ + --hash=sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595 \ + --hash=sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1 \ + --hash=sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678 \ + --hash=sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48 \ + --hash=sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76 \ + --hash=sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0 \ + --hash=sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18 \ + --hash=sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d \ + --hash=sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d \ + --hash=sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1 \ + --hash=sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981 \ + --hash=sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7 \ + --hash=sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82 \ + --hash=sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2 \ + --hash=sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4 \ + --hash=sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663 \ + --hash=sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c \ + --hash=sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d \ + --hash=sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a \ + --hash=sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a \ + --hash=sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d \ + --hash=sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b \ + --hash=sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a \ + --hash=sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826 \ + --hash=sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee \ + --hash=sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9 \ + --hash=sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648 \ + --hash=sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da \ + --hash=sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2 \ + --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2 \ + --hash=sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87 + # via + # apache-beam + # cloud-sql-python-connector + # google-auth + # secretstorage +distro==1.9.0 \ + --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 + # via google-genai +dnspython==2.8.0 \ + --hash=sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af \ + --hash=sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f + # via + # cloud-sql-python-connector + # pymongo +docstring-parser==0.17.0 \ + --hash=sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912 \ + --hash=sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708 + # via google-cloud-aiplatform +envoy-data-plane==0.2.6 \ + --hash=sha256:6341768b9cf5d6268baced4d2e8b3429f98664fbbe8958dae69ee25316ae869a \ + --hash=sha256:d1541c8cd00677886a2f93696edf9e3589cd4ac680defc66b3013ffb082f274c + # via apache-beam +fastavro==1.12.1 \ + --hash=sha256:00650ca533907361edda22e6ffe8cf87ab2091c5d8aee5c8000b0f2dcdda7ed3 \ + --hash=sha256:02281432dcb11c78b3280da996eff61ee0eff39c5de06c6e0fbf19275093e6d4 \ + --hash=sha256:0714b285160fcd515eb0455540f40dd6dac93bdeacdb03f24e8eac3d8aa51f8d \ + --hash=sha256:089e155c0c76e0d418d7e79144ce000524dd345eab3bc1e9c5ae69d500f71b14 \ + --hash=sha256:120aaf82ac19d60a1016afe410935fe94728752d9c2d684e267e5b7f0e70f6d9 \ + --hash=sha256:123fb221df3164abd93f2d042c82f538a1d5a43ce41375f12c91ce1355a9141e \ + --hash=sha256:1f55eef18c41d4476bd32a82ed5dd86aabc3f614e1b66bdb09ffa291612e1670 \ + --hash=sha256:1f81011d54dd47b12437b51dd93a70a9aa17b61307abf26542fc3c13efbc6c51 \ + --hash=sha256:2de72d786eb38be6b16d556b27232b1bf1b2797ea09599507938cdb7a9fe3e7c \ + --hash=sha256:2f285be49e45bc047ab2f6bed040bb349da85db3f3c87880e4b92595ea093b2b \ + --hash=sha256:3100ad643e7fa658469a2a2db229981c1a000ff16b8037c0b58ce3ec4d2107e8 \ + --hash=sha256:3616e2f0e1c9265e92954fa099db79c6e7817356d3ff34f4bcc92699ae99697c \ + --hash=sha256:3b1921ac35f3d89090a5816b626cf46e67dbecf3f054131f84d56b4e70496f45 \ + --hash=sha256:4128978b930aaf930332db4b3acc290783183f3be06a241ae4a482f3ed8ce892 \ + --hash=sha256:43ded16b3f4a9f1a42f5970c2aa618acb23ea59c4fcaa06680bdf470b255e5a8 \ + --hash=sha256:44cbff7518901c91a82aab476fcab13d102e4999499df219d481b9e15f61af34 \ + --hash=sha256:469fecb25cba07f2e1bfa4c8d008477cd6b5b34a59d48715e1b1a73f6160097d \ + --hash=sha256:509818cb24b98a804fc80be9c5fed90f660310ae3d59382fc811bfa187122167 \ + --hash=sha256:5217f773492bac43dae15ff2931432bce2d7a80be7039685a78d3fab7df910bd \ + --hash=sha256:546ffffda6610fca672f0ed41149808e106d8272bb246aa7539fa8bb6f117f17 \ + --hash=sha256:5aa777b8ee595b50aa084104cd70670bf25a7bbb9fd8bb5d07524b0785ee1699 \ + --hash=sha256:632a4e3ff223f834ddb746baae0cc7cee1068eb12c32e4d982c2fee8a5b483d0 \ + --hash=sha256:64961ab15b74b7c168717bbece5660e0f3d457837c3cc9d9145181d011199fa7 \ + --hash=sha256:6b632b713bc5d03928a87d811fa4a11d5f25cd43e79c161e291c7d3f7aa740fd \ + --hash=sha256:780476c23175d2ae457c52f45b9ffa9d504593499a36cd3c1929662bf5b7b14b \ + --hash=sha256:78df838351e4dff9edd10a1c41d1324131ffecbadefb9c297d612ef5363c049a \ + --hash=sha256:792356d320f6e757e89f7ac9c22f481e546c886454a6709247f43c0dd7058004 \ + --hash=sha256:81563e1f93570e6565487cdb01ba241a36a00e58cff9c5a0614af819d1155d8f \ + --hash=sha256:83e6caf4e7a8717d932a3b1ff31595ad169289bbe1128a216be070d3a8391671 \ + --hash=sha256:9090f0dee63fe022ee9cc5147483366cc4171c821644c22da020d6b48f576b4f \ + --hash=sha256:9445da127751ba65975d8e4bdabf36bfcfdad70fc35b2d988e3950cce0ec0e7c \ + --hash=sha256:a275e48df0b1701bb764b18a8a21900b24cf882263cb03d35ecdba636bbc830b \ + --hash=sha256:a38607444281619eda3a9c1be9f5397634012d1b237142eee1540e810b30ac8b \ + --hash=sha256:a7d840ccd9aacada3ddc80fbcc4ea079b658107fe62e9d289a0de9d54e95d366 \ + --hash=sha256:a8bc2dcec5843d499f2489bfe0747999108f78c5b29295d877379f1972a3d41a \ + --hash=sha256:ac76d6d95f909c72ee70d314b460b7e711d928845771531d823eb96a10952d26 \ + --hash=sha256:b6a3462934b20a74f9ece1daa49c2e4e749bd9a35fa2657b53bf62898fba80f5 \ + --hash=sha256:b81fc04e85dfccf7c028e0580c606e33aa8472370b767ef058aae2c674a90746 \ + --hash=sha256:b91a0fe5a173679a6c02d53ca22dcaad0a2c726b74507e0c1c2e71a7c3f79ef9 \ + --hash=sha256:bec207360f76f0b3de540758a297193c5390e8e081c43c3317f610b1414d8c8f \ + --hash=sha256:c0390bfe4a9f8056a75ac6785fbbff8f5e317f5356481d2e29ec980877d2314b \ + --hash=sha256:c3d67c47f177e486640404a56f2f50b165fe892cc343ac3a34673b80cc7f1dd6 \ + --hash=sha256:cb0337b42fd3c047fcf0e9b7597bd6ad25868de719f29da81eabb6343f08d399 \ + --hash=sha256:d71c8aa841ef65cfab709a22bb887955f42934bced3ddb571e98fdbdade4c609 \ + --hash=sha256:eaa7ab3769beadcebb60f0539054c7755f63bd9cf7666e2c15e615ab605f89a8 \ + --hash=sha256:ed924233272719b5d5a6a0b4d80ef3345fc7e84fc7a382b6232192a9112d38a6 + # via apache-beam +fasteners==0.20 \ + --hash=sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8 \ + --hash=sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7 + # via + # apache-beam + # google-apitools +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal +google-api-core[grpc]==2.30.0 \ + --hash=sha256:02edfa9fab31e17fc0befb5f161b3bf93c9096d99aed584625f38065c511ad9b \ + --hash=sha256:80be49ee937ff9aba0fd79a6eddfde35fe658b9953ab9b79c57dd7061afa8df5 + # via + # apache-beam + # google-cloud-aiplatform + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-kms + # google-cloud-language + # google-cloud-monitoring + # google-cloud-pubsub + # google-cloud-pubsublite + # google-cloud-recommendations-ai + # google-cloud-resource-manager + # google-cloud-secret-manager + # google-cloud-spanner + # google-cloud-storage + # google-cloud-videointelligence + # google-cloud-vision +google-apitools==0.5.31 \ + --hash=sha256:4af0dd6dd4582810690251f0b57a97c1873dadfda54c5bc195844c8907624170 \ + --hash=sha256:6be92c1c3e93485450420bb0e365d47eb4d8a835d03ebe1963dc6da4d39a7b0e + # via apache-beam +google-auth[requests]==2.49.0 \ + --hash=sha256:9cc2d9259d3700d7a257681f81052db6737495a1a46b610597f4b8bafe5286ae \ + --hash=sha256:f893ef7307f19cf53700b7e2f61b5a6affe3aa0edf9943b13788920ab92d8d87 + # via + # apache-beam + # cloud-sql-python-connector + # google-api-core + # google-auth-httplib2 + # google-cloud-aiplatform + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-kms + # google-cloud-language + # google-cloud-monitoring + # google-cloud-pubsub + # google-cloud-recommendations-ai + # google-cloud-resource-manager + # google-cloud-secret-manager + # google-cloud-storage + # google-cloud-videointelligence + # google-cloud-vision + # google-genai + # keyrings-google-artifactregistry-auth +google-auth-httplib2==0.2.1 \ + --hash=sha256:1be94c611db91c01f9703e7f62b0a59bbd5587a95571c7b6fade510d648bc08b \ + --hash=sha256:5ef03be3927423c87fb69607b42df23a444e434ddb2555b73b3679793187b7de + # via apache-beam +google-cloud-aiplatform==1.140.0 \ + --hash=sha256:e94493a2682b9d17efa7146a53bb3665bf1595c3394fd3d0f45d18f71623fddc \ + --hash=sha256:ea7eb1870b4cf600f8c2472102e21c3a1bcaf723d6e49f00ed51bc6b88d54fff + # via apache-beam +google-cloud-bigquery==3.40.1 \ + --hash=sha256:75afcfb6e007238fe1deefb2182105249321145ff921784fe7b1de2b4ba24506 \ + --hash=sha256:9082a6b8193aba87bed6a2c79cf1152b524c99bb7e7ac33a785e333c09eac868 + # via + # apache-beam + # google-cloud-aiplatform +google-cloud-bigquery-storage==2.36.2 \ + --hash=sha256:823a73db0c4564e8ad3eedcfd5049f3d5aa41775267863b5627211ec36be2dbf \ + --hash=sha256:ad49d8c09ad6cd82da4efe596fcfcdbc1458bf05b93915e3c5c00f1e700ae128 + # via + # -r python/default_base_bqmonitor_requirements.txt + # apache-beam +google-cloud-bigtable==2.35.0 \ + --hash=sha256:f355bfce1f239453ec2bb3839b0f4f9937cf34ef06ef29e1ca63d58fd38d0c50 \ + --hash=sha256:f5699012c5fea4bd4bdf7e80e5e3a812a847eb8f41bf8dc2f43095d6d876b83b + # via apache-beam +google-cloud-core==2.5.0 \ + --hash=sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc \ + --hash=sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963 + # via + # apache-beam + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-spanner + # google-cloud-storage +google-cloud-datastore==2.23.0 \ + --hash=sha256:24a1b1d29b902148fe41b109699f76fd3aa60591e9d547c0f8b87d7bf9ff213f \ + --hash=sha256:80049883a4ae928fdcc661ba6803ec267665dc0e6f3ce2da91441079a6bb6387 + # via apache-beam +google-cloud-dlp==3.34.0 \ + --hash=sha256:3a1a7fd335fd65641ac3cb3f24f96ee9345d546d413ad6c88071a59404b1a641 \ + --hash=sha256:6dfa3172520d5a7fa8ccce47a9622cde815f037b4aa6fb6d69984fd597bf8007 + # via apache-beam +google-cloud-kms==3.11.0 \ + --hash=sha256:07f2829e4ed986220802d013219fe159ecbdecec35907a6ddeea37ea9daecd8d \ + --hash=sha256:5f7d7bdb347f13a8a2b7bad6cbdf3846a51690df7215586845b62851b88839f7 + # via apache-beam +google-cloud-language==2.19.0 \ + --hash=sha256:3b88f6eabd1c2413a1c6c918cbe40a22a5d14401930309717dbb709b353c6c64 \ + --hash=sha256:a43044632c8aada30a9c3246e00bfc867a56188be0c6e08e8764731296a05e0b + # via apache-beam +google-cloud-monitoring==2.29.1 \ + --hash=sha256:86cac55cdd2608561819d19544fb3c129bbb7dcecc445d8de426e34cd6fa8e49 \ + --hash=sha256:944a57031f20da38617d184d5658c1f938e019e8061f27fd944584831a1b9d5a + # via google-cloud-spanner +google-cloud-pubsub==2.35.0 \ + --hash=sha256:2c0d1d7ccda52fa12fb73f34b7eb9899381e2fd931c7d47b10f724cdfac06f95 \ + --hash=sha256:c32e4eb29e532ec784b5abb5d674807715ec07895b7c022b9404871dec09970d + # via + # apache-beam + # google-cloud-pubsublite +google-cloud-pubsublite==1.13.0 \ + --hash=sha256:00773be42f335ec0e76e0e3e6c72041c2795268433f48add29780cea41e8bd3e \ + --hash=sha256:cc56ca57755e7665a66f0c0025ca923f7bfeb39ba408859ffe87cb840c0e82b5 + # via apache-beam +google-cloud-recommendations-ai==0.10.18 \ + --hash=sha256:a6bccb45744fd89f038aa3e19502d1f46ea61c438dd2c08528533f8e185ec469 \ + --hash=sha256:c5c4b569d8be96e65dc273d18a35e44147ef62f845c8a9e8afd93474802c60c8 + # via apache-beam +google-cloud-resource-manager==1.16.0 \ + --hash=sha256:cc938f87cc36c2672f062b1e541650629e0d954c405a4dac35ceedee70c267c3 \ + --hash=sha256:fb9a2ad2b5053c508e1c407ac31abfd1a22e91c32876c1892830724195819a28 + # via google-cloud-aiplatform +google-cloud-secret-manager==2.26.0 \ + --hash=sha256:0d1d6f76327685a0ed78a4cf50f289e1bfbbe56026ed0affa98663b86d6d50d6 \ + --hash=sha256:940a5447a6ec9951446fd1a0f22c81a4303fde164cd747aae152c5f5c8e6723e + # via apache-beam +google-cloud-spanner==3.63.0 \ + --hash=sha256:6ffae0ed589bbbd2d8831495e266198f3d069005cfe65c664448c9a727c88e7b \ + --hash=sha256:e2a4fb3bdbad4688645f455d498705d3f935b7c9011f5c94c137b77569b47a62 + # via apache-beam +google-cloud-storage==2.19.0 \ + --hash=sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba \ + --hash=sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2 + # via + # apache-beam + # google-cloud-aiplatform +google-cloud-videointelligence==2.18.0 \ + --hash=sha256:2cf4a32f64f4e01fdfb78b7bf625aa82df9129c87854796348887eac60290e95 \ + --hash=sha256:b2ae39bd22d186218684a297c2fa2fa636e5874e69d39f719504d729f44639fd + # via apache-beam +google-cloud-vision==3.12.1 \ + --hash=sha256:8c661bc0e7a6bd3d03a1a645b977af24ae3f21ccf3df8e213298659fd0d40813 \ + --hash=sha256:f99b83af7588d30e708b87e09ff73e43e380497fe82c799b9f05e03f310027c8 + # via apache-beam +google-crc32c==1.8.0 \ + --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ + --hash=sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a \ + --hash=sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff \ + --hash=sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288 \ + --hash=sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411 \ + --hash=sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a \ + --hash=sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15 \ + --hash=sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb \ + --hash=sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa \ + --hash=sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962 \ + --hash=sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215 \ + --hash=sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b \ + --hash=sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27 \ + --hash=sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113 \ + --hash=sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f \ + --hash=sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f \ + --hash=sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d \ + --hash=sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2 \ + --hash=sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092 \ + --hash=sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7 \ + --hash=sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2 \ + --hash=sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93 \ + --hash=sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8 \ + --hash=sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21 \ + --hash=sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79 \ + --hash=sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2 \ + --hash=sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc \ + --hash=sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454 \ + --hash=sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2 \ + --hash=sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733 \ + --hash=sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697 \ + --hash=sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651 \ + --hash=sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c + # via + # google-cloud-bigtable + # google-cloud-storage + # google-resumable-media +google-genai==1.66.0 \ + --hash=sha256:7f127a39cf695277104ce4091bb26e417c59bb46e952ff3699c3a982d9c474ee \ + --hash=sha256:ffc01647b65046bca6387320057aa51db0ad64bcc72c8e3e914062acfa5f7c49 + # via google-cloud-aiplatform +google-resumable-media==2.8.0 \ + --hash=sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582 \ + --hash=sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos[grpc]==1.73.0 \ + --hash=sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a \ + --hash=sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8 + # via + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +grpc-google-iam-v1==0.14.3 \ + --hash=sha256:7a7f697e017a067206a3dfef44e4c634a34d3dee135fe7d7a4613fe3e59217e6 \ + --hash=sha256:879ac4ef33136c5491a6300e27575a9ec760f6cdf9a2518798c1b8977a5dc389 + # via + # google-cloud-bigtable + # google-cloud-kms + # google-cloud-pubsub + # google-cloud-resource-manager + # google-cloud-secret-manager + # google-cloud-spanner +grpc-interceptor==0.15.4 \ + --hash=sha256:0035f33228693ed3767ee49d937bac424318db173fef4d2d0170b3215f254d9d \ + --hash=sha256:1f45c0bcb58b6f332f37c637632247c9b02bc6af0fdceb7ba7ce8d2ebbfb0926 + # via google-cloud-spanner +grpcio==1.65.5 \ + --hash=sha256:05f02d68fc720e085f061b704ee653b181e6d5abfe315daef085719728d3d1fd \ + --hash=sha256:078038e150a897e5e402ed3d57f1d31ebf604cbed80f595bd281b5da40762a92 \ + --hash=sha256:0b2944390a496567de9e70418f3742b477d85d8ca065afa90432edc91b4bb8ad \ + --hash=sha256:11f8b16121768c1cb99d7dcb84e01510e60e6a206bf9123e134118802486f035 \ + --hash=sha256:1c4caafe71aef4dabf53274bbf4affd6df651e9f80beedd6b8e08ff438ed3260 \ + --hash=sha256:1cbc208edb9acf1cc339396a1a36b83796939be52f34e591c90292045b579fbf \ + --hash=sha256:238a625f391a1b9f5f069bdc5930f4fd71b74426bea52196fc7b83f51fa97d34 \ + --hash=sha256:2a6d8169812932feac514b420daffae8ab8e36f90f3122b94ae767e633296b17 \ + --hash=sha256:2b91ce647b6307f25650872454a4d02a2801f26a475f90d0b91ed8110baae589 \ + --hash=sha256:3207ae60d07e5282c134b6e02f9271a2cb523c6d7a346c6315211fe2bf8d61ed \ + --hash=sha256:32d60e18ff7c34fe3f6db3d35ad5c6dc99f5b43ff3982cb26fad4174462d10b1 \ + --hash=sha256:33158e56c6378063923c417e9fbdb28660b6e0e2835af42e67f5a7793f587af7 \ + --hash=sha256:47d0aaaab82823f0aa6adea5184350b46e2252e13a42a942db84da5b733f2e05 \ + --hash=sha256:55714ea852396ec9568f45f487639945ab674de83c12bea19d5ddbc3ae41ada3 \ + --hash=sha256:6c4e62bcf297a1568f627f39576dbfc27f1e5338a691c6dd5dd6b3979da51d1c \ + --hash=sha256:76991b7a6fb98630a3328839755181ce7c1aa2b1842aa085fd4198f0e5198960 \ + --hash=sha256:770bd4bd721961f6dd8049bc27338564ba8739913f77c0f381a9815e465ff965 \ + --hash=sha256:7a412959aa5f08c5ac04aa7b7c3c041f5e4298cadd4fcc2acff195b56d185ebc \ + --hash=sha256:84c901cdec16a092099f251ef3360d15e29ef59772150fa261d94573612539b5 \ + --hash=sha256:85ae8f8517d5bcc21fb07dbf791e94ed84cc28f84c903cdc2bd7eaeb437c8f45 \ + --hash=sha256:89c00a18801b1ed9cc441e29b521c354725d4af38c127981f2c950c796a09b6e \ + --hash=sha256:8da58ff80bc4556cf29bc03f5fff1f03b8387d6aaa7b852af9eb65b2cf833be4 \ + --hash=sha256:8e5c4c15ac3fe1eb68e46bc51e66ad29be887479f231f8237cf8416058bf0cc1 \ + --hash=sha256:a101696f9ece90a0829988ff72f1b1ea2358f3df035bdf6d675dd8b60c2c0894 \ + --hash=sha256:a2f80510f99f82d4eb825849c486df703f50652cea21c189eacc2b84f2bde764 \ + --hash=sha256:a70a20eed87bba647a38bedd93b3ce7db64b3f0e8e0952315237f7f5ca97b02d \ + --hash=sha256:a80e9a5e3f93c54f5eb82a3825ea1fc4965b2fa0026db2abfecb139a5c4ecdf1 \ + --hash=sha256:ab5ec837d8cee8dbce9ef6386125f119b231e4333cc6b6d57b6c5c7c82a72331 \ + --hash=sha256:b67d450f1e008fedcd81e097a3a400a711d8be1a8b20f852a7b8a73fead50fe3 \ + --hash=sha256:b7ca419f1462390851eec395b2089aad1e49546b52d4e2c972ceb76da69b10f8 \ + --hash=sha256:b8270b15b99781461b244f5c81d5c2bc9696ab9189fb5ff86c841417fb3b39fe \ + --hash=sha256:bc74f3f745c37e2c5685c9d2a2d5a94de00f286963f5213f763ae137bf4f2358 \ + --hash=sha256:c3655139d7be213c32c79ef6fb2367cae28e56ef68e39b1961c43214b457f257 \ + --hash=sha256:c97962720489ef31b5ad8a916e22bc31bba3664e063fb9f6702dce056d4aa61b \ + --hash=sha256:cabd706183ee08d8026a015af5819a0b3a8959bdc9d1f6fdacd1810f09200f2a \ + --hash=sha256:d3a9e35bcb045e39d7cac30464c285389b9a816ac2067e4884ad2c02e709ef8e \ + --hash=sha256:d750e9330eb14236ca11b78d0c494eed13d6a95eb55472298f0e547c165ee324 \ + --hash=sha256:d7df567b67d16d4177835a68d3f767bbcbad04da9dfb52cbd19171f430c898bd \ + --hash=sha256:ec6f219fb5d677a522b0deaf43cea6697b16f338cb68d009e30930c4aa0d2209 \ + --hash=sha256:ec71fc5b39821ad7d80db7473c8f8c2910f3382f0ddadfbcfc2c6c437107eb67 \ + --hash=sha256:ee6ed64a27588a2c94e8fa84fe8f3b5c89427d4d69c37690903d428ec61ca7e4 \ + --hash=sha256:f17f9fa2d947dbfaca01b3ab2c62eefa8240131fdc67b924eb42ce6032e3e5c1 \ + --hash=sha256:f5b5970341359341d0e4c789da7568264b2a89cd976c05ea476036852b5950cd \ + --hash=sha256:f79c87c114bf37adf408026b9e2e333fe9ff31dfc9648f6f80776c513145c813 \ + --hash=sha256:fa36dd8496d3af0d40165252a669fa4f6fd2db4b4026b9a9411cbf060b9d6a15 \ + --hash=sha256:fe6505376f5b00bb008e4e1418152e3ad3d954b629da286c7913ff3cfc0ff740 + # via + # apache-beam + # google-api-core + # google-cloud-bigquery-storage + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-kms + # google-cloud-language + # google-cloud-monitoring + # google-cloud-pubsub + # google-cloud-pubsublite + # google-cloud-resource-manager + # google-cloud-secret-manager + # google-cloud-videointelligence + # google-cloud-vision + # googleapis-common-protos + # grpc-google-iam-v1 + # grpc-interceptor + # grpcio-status +grpcio-status==1.65.5 \ + --hash=sha256:2c9fa3af32efd26f01837d44305dce106973bc5357b9a9fc8bbd87bb8bf833d1 \ + --hash=sha256:44a445ce55375545a913e005be36fbec7999a4cc320d7aecb7a4469d3d49366c + # via + # google-api-core + # google-cloud-pubsub + # google-cloud-pubsublite +grpclib==0.4.9 \ + --hash=sha256:7762ec1c8ed94dfad597475152dd35cbd11aecaaca2f243e29702435ca24cf0e \ + --hash=sha256:cc589c330fa81004c6400a52a566407574498cb5b055fa927013361e21466c46 + # via betterproto +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +h2==4.3.0 \ + --hash=sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1 \ + --hash=sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd + # via grpclib +hpack==4.1.0 \ + --hash=sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496 \ + --hash=sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca + # via h2 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httplib2==0.22.0 \ + --hash=sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc \ + --hash=sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81 + # via + # apache-beam + # google-apitools + # google-auth-httplib2 + # oauth2client +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via google-genai +hyperframe==6.1.0 \ + --hash=sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5 \ + --hash=sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08 + # via h2 +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via + # anyio + # httpx + # requests + # yarl +importlib-metadata==8.7.1 \ + --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \ + --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151 + # via + # keyring + # opentelemetry-api +jaraco-classes==3.4.0 \ + --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 + # via keyring +jaraco-context==6.1.1 \ + --hash=sha256:0df6a0287258f3e364072c3e40d5411b20cafa30cb28c4839d24319cecf9f808 \ + --hash=sha256:bc046b2dc94f1e5532bd02402684414575cc11f565d929b6563125deb0a6e581 + # via keyring +jaraco-functools==4.4.0 \ + --hash=sha256:9eec1e36f45c818d9bf307c8948eb03b2b56cd44087b3cdc989abca1f20b9176 \ + --hash=sha256:da21933b0417b89515562656547a77b4931f98176eb173644c0d35032a33d6bb + # via keyring +jeepney==0.9.0 \ + --hash=sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683 \ + --hash=sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732 + # via + # keyring + # secretstorage +jsonpickle==3.4.2 \ + --hash=sha256:2efa2778859b6397d5804b0a98d52cd2a7d9a70fcb873bc5a3ca5acca8f499ba \ + --hash=sha256:fd6c273278a02b3b66e3405db3dd2f4dbc8f4a4a3123bfcab3045177c6feb9c3 + # via apache-beam +keyring==25.7.0 \ + --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f \ + --hash=sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b + # via keyrings-google-artifactregistry-auth +keyrings-google-artifactregistry-auth==1.1.2 \ + --hash=sha256:bd6abb72740d2dfeb4a5c03c3b105c6f7dba169caa29dee3959694f1f02c77de \ + --hash=sha256:e3f18b50fa945c786593014dc225810d191671d4f5f8e12d9259e39bad3605a3 + # via apache-beam +mmh3==5.2.1 \ + --hash=sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d \ + --hash=sha256:0634581290e6714c068f4aa24020acf7880927d1f0084fa753d9799ae9610082 \ + --hash=sha256:08043f7cb1fb9467c3fbbbaea7896986e7fbc81f4d3fd9289a73d9110ab6207a \ + --hash=sha256:0a3984146e414684a6be2862d84fcb1035f4984851cb81b26d933bab6119bf00 \ + --hash=sha256:0bbc17250b10d3466875a40a52520a6bac3c02334ca709207648abd3c223ed5c \ + --hash=sha256:0cc21533878e5586b80d74c281d7f8da7932bc8ace50b8d5f6dbf7e3935f63f1 \ + --hash=sha256:0d0b7e803191db5f714d264044e06189c8ccd3219e936cc184f07106bd17fd7b \ + --hash=sha256:113f78e7463a36dbbcea05bfe688efd7fa759d0f0c56e73c974d60dcfec3dfcc \ + --hash=sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104 \ + --hash=sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8 \ + --hash=sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9 \ + --hash=sha256:1d9f9a3ce559a5267014b04b82956993270f63ec91765e13e9fd73daf2d2738e \ + --hash=sha256:1e4ecee40ba19e6975e1120829796770325841c2f153c0e9aecca927194c6a2a \ + --hash=sha256:22b0f9971ec4e07e8223f2beebe96a6cfc779d940b6f27d26604040dd74d3a44 \ + --hash=sha256:26fb5b9c3946bf7f1daed7b37e0c03898a6f062149127570f8ede346390a0825 \ + --hash=sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4 \ + --hash=sha256:28cfab66577000b9505a0d068c731aee7ca85cd26d4d63881fab17857e0fe1fb \ + --hash=sha256:29bc3973676ae334412efdd367fcd11d036b7be3efc1ce2407ef8676dabfeb82 \ + --hash=sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f \ + --hash=sha256:2d5d542bf2abd0fd0361e8017d03f7cb5786214ceb4a40eef1539d6585d93386 \ + --hash=sha256:30e4d2084df019880d55f6f7bea35328d9b464ebee090baa372c096dc77556fb \ + --hash=sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d \ + --hash=sha256:368625fb01666655985391dbad3860dc0ba7c0d6b9125819f3121ee7292b4ac8 \ + --hash=sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593 \ + --hash=sha256:3a9fed49c6ce4ed7e73f13182760c65c816da006debe67f37635580dfb0fae00 \ + --hash=sha256:3c38d142c706201db5b2345166eeef1e7740e3e2422b470b8ba5c8727a9b4c7a \ + --hash=sha256:3cb61db880ec11e984348227b333259994c2c85caa775eb7875decb3768db890 \ + --hash=sha256:3d74a03fb57757ece25aa4b3c1c60157a1cece37a020542785f942e2f827eed5 \ + --hash=sha256:3f796b535008708846044c43302719c6956f39ca2d93f2edda5319e79a29efbb \ + --hash=sha256:41105377f6282e8297f182e393a79cfffd521dde37ace52b106373bdcd9ca5cb \ + --hash=sha256:41aac7002a749f08727cb91babff1daf8deac317c0b1f317adc69be0e6c375d1 \ + --hash=sha256:44983e45310ee5b9f73397350251cdf6e63a466406a105f1d16cb5baa659270b \ + --hash=sha256:4cbbde66f1183db040daede83dd86c06d663c5bb2af6de1142b7c8c37923dd74 \ + --hash=sha256:4eda76074cfca2787c8cf1bec603eaebdddd8b061ad5502f85cddae998d54f00 \ + --hash=sha256:4fc6cd65dc4d2fdb2625e288939a3566e36127a84811a4913f02f3d5931da52d \ + --hash=sha256:50885073e2909251d4718634a191c49ae5f527e5e1736d738e365c3e8be8f22b \ + --hash=sha256:5174a697ce042fa77c407e05efe41e03aa56dae9ec67388055820fb48cf4c3ba \ + --hash=sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b \ + --hash=sha256:54fe8518abe06a4c3852754bfd498b30cc58e667f376c513eac89a244ce781a4 \ + --hash=sha256:55dbbd8ffbc40d1697d5e2d0375b08599dae8746b0b08dea05eee4ce81648fac \ + --hash=sha256:57b52603e89355ff318025dd55158f6e71396c0f1f609d548e9ea9c94cc6ce0a \ + --hash=sha256:58370d05d033ee97224c81263af123dea3d931025030fd34b61227a768a8858a \ + --hash=sha256:5d87a3584093e1a89987e3d36d82c98d9621b2cb944e22a420aa1401e096758f \ + --hash=sha256:623f938f6a039536cc02b7582a07a080f13fdfd48f87e63201d92d7e34d09a18 \ + --hash=sha256:62815d2c67f2dd1be76a253d88af4e1da19aeaa1820146dec52cf8bee2958b16 \ + --hash=sha256:6290289fa5fb4c70fd7f72016e03633d60388185483ff3b162912c81205ae2cf \ + --hash=sha256:67e41a497bac88cc1de96eeba56eeb933c39d54bc227352f8455aa87c4ca4000 \ + --hash=sha256:6c85c38a279ca9295a69b9b088a2e48aa49737bb1b34e6a9dc6297c110e8d912 \ + --hash=sha256:6f01f044112d43a20be2f13a11683666d87151542ad627fe41a18b9791d2802f \ + --hash=sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5 \ + --hash=sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e \ + --hash=sha256:72d1cc63bcc91e14933f77d51b3df899d6a07d184ec515ea7f56bff659e124d7 \ + --hash=sha256:7374d6e3ef72afe49697ecd683f3da12f4fc06af2d75433d0580c6746d2fa025 \ + --hash=sha256:7501e9be34cb21e72fcfe672aafd0eee65c16ba2afa9dcb5500a587d3a0580f0 \ + --hash=sha256:76219cd1eefb9bf4af7856e3ae563d15158efa145c0aab01e9933051a1954045 \ + --hash=sha256:7aec798c2b01aaa65a55f1124f3405804184373abb318a3091325aece235f67c \ + --hash=sha256:7be6dfb49e48fd0a7d91ff758a2b51336f1cd21f9d44b20f6801f072bd080cdd \ + --hash=sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d \ + --hash=sha256:7e8ec5f606e0809426d2440e0683509fb605a8820a21ebd120dcdba61b74ef7f \ + --hash=sha256:7f196cd7910d71e9d9860da0ff7a77f64d22c1ad931f1dd18559a06e03109fc0 \ + --hash=sha256:82f3802bfc4751f420d591c5c864de538b71cea117fce67e4595c2afede08a15 \ + --hash=sha256:85ffc9920ffc39c5eee1e3ac9100c913a0973996fbad5111f939bbda49204bb7 \ + --hash=sha256:8e6c219e375f6341d0959af814296372d265a8ca1af63825f65e2e87c618f006 \ + --hash=sha256:8f767ba0911602ddef289404e33835a61168314ebd3c729833db2ed685824211 \ + --hash=sha256:8ff038d52ef6aa0f309feeba00c5095c9118d0abf787e8e8454d6048db2037fc \ + --hash=sha256:915e7a2418f10bd1151b1953df06d896db9783c9cfdb9a8ee1f9b3a4331ab503 \ + --hash=sha256:92883836caf50d5255be03d988d75bc93e3f86ba247b7ca137347c323f731deb \ + --hash=sha256:960b1b3efa39872ac8b6cc3a556edd6fb90ed74f08c9c45e028f1005b26aa55d \ + --hash=sha256:9aeaf53eaa075dd63e81512522fd180097312fb2c9f476333309184285c49ce0 \ + --hash=sha256:9d8089d853c7963a8ce87fff93e2a67075c0bc08684a08ea6ad13577c38ffc38 \ + --hash=sha256:a4130d0b9ce5fad6af07421b1aecc7e079519f70d6c05729ab871794eded8617 \ + --hash=sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f \ + --hash=sha256:add7ac388d1e0bf57259afbcf9ed05621a3bf11ce5ee337e7536f1e1aaf056b0 \ + --hash=sha256:b1f12bd684887a0a5d55e6363ca87056f361e45451105012d329b86ec19dbe0b \ + --hash=sha256:b3f99e1756fc48ad507b95e5d86f2fb21b3d495012ff13e6592ebac14033f166 \ + --hash=sha256:b4cce60d0223074803c9dbe0721ad3fa51dafe7d462fee4b656a1aa01ee07518 \ + --hash=sha256:baeb47635cb33375dee4924cd93d7f5dcaa786c740b08423b0209b824a1ee728 \ + --hash=sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad \ + --hash=sha256:bbfcb95d9a744e6e2827dfc66ad10e1020e0cac255eb7f85652832d5a264c2fc \ + --hash=sha256:bd6e7d363aa93bd3421b30b6af97064daf47bc96005bddba67c5ffbc6df426b8 \ + --hash=sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03 \ + --hash=sha256:c302245fd6c33d96bd169c7ccf2513c20f4c1e417c07ce9dce107c8bc3f8411f \ + --hash=sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2 \ + --hash=sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229 \ + --hash=sha256:cd471ede0d802dd936b6fab28188302b2d497f68436025857ca72cd3810423fe \ + --hash=sha256:d106493a60dcb4aef35a0fac85105e150a11cf8bc2b0d388f5a33272d756c966 \ + --hash=sha256:d30b650595fdbe32366b94cb14f30bb2b625e512bd4e1df00611f99dc5c27fd4 \ + --hash=sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6 \ + --hash=sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1 \ + --hash=sha256:d771f085fcdf4035786adfb1d8db026df1eb4b41dac1c3d070d1e49512843227 \ + --hash=sha256:dae0f0bd7d30c0ad61b9a504e8e272cb8391eed3f1587edf933f4f6b33437450 \ + --hash=sha256:db0562c5f71d18596dcd45e854cf2eeba27d7543e1a3acdafb7eef728f7fe85d \ + --hash=sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b \ + --hash=sha256:e080c0637aea036f35507e803a4778f119a9b436617694ae1c5c366805f1e997 \ + --hash=sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6 \ + --hash=sha256:e8b4b5580280b9265af3e0409974fb79c64cf7523632d03fbf11df18f8b0181e \ + --hash=sha256:e8b5378de2b139c3a830f0209c1e91f7705919a4b3e563a10955104f5097a70a \ + --hash=sha256:e904f2417f0d6f6d514f3f8b836416c360f306ddaee1f84de8eef1e722d212e5 \ + --hash=sha256:eee884572b06bbe8a2b54f424dbd996139442cf83c76478e1ec162512e0dd2c7 \ + --hash=sha256:f1fbb0a99125b1287c6d9747f937dc66621426836d1a2d50d05aecfc81911b57 \ + --hash=sha256:f40a95186a72fa0b67d15fef0f157bfcda00b4f59c8a07cbe5530d41ac35d105 \ + --hash=sha256:f6e0bfe77d238308839699944164b96a2eeccaf55f2af400f54dc20669d8d5f2 \ + --hash=sha256:f963eafc0a77a6c0562397da004f5876a9bcf7265a7bcc3205e29636bc4a1312 \ + --hash=sha256:fb9d44c25244e11c8be3f12c938ca8ba8404620ef8092245d2093c6ab3df260f \ + --hash=sha256:fc78739b5ec6e4fb02301984a3d442a91406e7700efbe305071e7fd1c78278f2 \ + --hash=sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a \ + --hash=sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b + # via google-cloud-spanner +more-itertools==10.8.0 \ + --hash=sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b \ + --hash=sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd + # via + # jaraco-classes + # jaraco-functools +multidict==6.7.1 \ + --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ + --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ + --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \ + --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \ + --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \ + --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \ + --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \ + --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \ + --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \ + --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \ + --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \ + --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \ + --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \ + --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \ + --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \ + --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \ + --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \ + --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \ + --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \ + --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \ + --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \ + --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \ + --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \ + --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \ + --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \ + --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \ + --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \ + --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \ + --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \ + --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \ + --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \ + --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \ + --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \ + --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \ + --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \ + --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \ + --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \ + --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \ + --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \ + --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \ + --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \ + --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \ + --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \ + --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \ + --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \ + --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \ + --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \ + --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \ + --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \ + --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \ + --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \ + --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \ + --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \ + --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \ + --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \ + --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \ + --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \ + --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \ + --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \ + --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \ + --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \ + --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \ + --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \ + --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \ + --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \ + --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \ + --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \ + --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \ + --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \ + --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \ + --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \ + --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \ + --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \ + --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \ + --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \ + --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \ + --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \ + --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \ + --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \ + --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \ + --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \ + --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \ + --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \ + --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \ + --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \ + --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \ + --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \ + --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \ + --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \ + --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \ + --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \ + --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \ + --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \ + --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \ + --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \ + --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \ + --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \ + --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \ + --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \ + --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \ + --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \ + --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \ + --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \ + --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \ + --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \ + --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \ + --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \ + --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \ + --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \ + --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \ + --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \ + --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \ + --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \ + --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \ + --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \ + --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \ + --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \ + --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \ + --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \ + --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \ + --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \ + --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \ + --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \ + --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \ + --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \ + --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \ + --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \ + --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \ + --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \ + --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \ + --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \ + --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \ + --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \ + --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \ + --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \ + --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \ + --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \ + --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \ + --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \ + --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \ + --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \ + --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \ + --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \ + --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \ + --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \ + --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19 + # via + # aiohttp + # grpclib + # yarl +numpy==2.4.2 \ + --hash=sha256:00ab83c56211a1d7c07c25e3217ea6695e50a3e2f255053686b081dc0b091a82 \ + --hash=sha256:068cdb2d0d644cdb45670810894f6a0600797a69c05f1ac478e8d31670b8ee75 \ + --hash=sha256:0f01dcf33e73d80bd8dc0f20a71303abbafa26a19e23f6b68d1aa9990af90257 \ + --hash=sha256:0fece1d1f0a89c16b03442eae5c56dc0be0c7883b5d388e0c03f53019a4bfd71 \ + --hash=sha256:12e26134a0331d8dbd9351620f037ec470b7c75929cb8a1537f6bfe411152a1a \ + --hash=sha256:1ae241bbfc6ae276f94a170b14785e561cb5e7f626b6688cf076af4110887413 \ + --hash=sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181 \ + --hash=sha256:209fae046e62d0ce6435fcfe3b1a10537e858249b3d9b05829e2a05218296a85 \ + --hash=sha256:20abd069b9cda45874498b245c8015b18ace6de8546bf50dfa8cea1696ed06ef \ + --hash=sha256:21982668592194c609de53ba4933a7471880ccbaadcc52352694a59ecc860b3a \ + --hash=sha256:25f2059807faea4b077a2b6837391b5d830864b3543627f381821c646f31a63c \ + --hash=sha256:2653de5c24910e49c2b106499803124dde62a5a1fe0eedeaecf4309a5f639390 \ + --hash=sha256:2b8f157c8a6f20eb657e240f8985cc135598b2b46985c5bccbde7616dc9c6b1e \ + --hash=sha256:2fb882da679409066b4603579619341c6d6898fc83a8995199d5249f986e8e8f \ + --hash=sha256:40397bda92382fcec844066efb11f13e1c9a3e2a8e8f318fb72ed8b6db9f60f1 \ + --hash=sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b \ + --hash=sha256:47c5a6ed21d9452b10227e5e8a0e1c22979811cad7dcc19d8e3e2fb8fa03f1a3 \ + --hash=sha256:4f069069931240b3fc703f1e23df63443dbd6390614c8c44a87d96cd0ec81eb1 \ + --hash=sha256:52b913ec40ff7ae845687b0b34d8d93b60cb66dcee06996dd5c99f2fc9328657 \ + --hash=sha256:5633c0da313330fd20c484c78cdd3f9b175b55e1a766c4a174230c6b70ad8262 \ + --hash=sha256:5daf6f3914a733336dab21a05cdec343144600e964d2fcdabaac0c0269874b2a \ + --hash=sha256:5eea80d908b2c1f91486eb95b3fb6fab187e569ec9752ab7d9333d2e66bf2d6b \ + --hash=sha256:602f65afdef699cda27ec0b9224ae5dc43e328f4c24c689deaf77133dbee74d0 \ + --hash=sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae \ + --hash=sha256:66cb9422236317f9d44b67b4d18f44efe6e9c7f8794ac0462978513359461554 \ + --hash=sha256:6d82351358ffbcdcd7b686b90742a9b86632d6c1c051016484fa0b326a0a1548 \ + --hash=sha256:6e9f61981ace1360e42737e2bae58b27bf28a1b27e781721047d84bd754d32e7 \ + --hash=sha256:6ed0be1ee58eef41231a5c943d7d1375f093142702d5723ca2eb07db9b934b05 \ + --hash=sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1 \ + --hash=sha256:7df2de1e4fba69a51c06c28f5a3de36731eb9639feb8e1cf7e4a7b0daf4cf622 \ + --hash=sha256:7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1 \ + --hash=sha256:7f54844851cdb630ceb623dcec4db3240d1ac13d4990532446761baede94996a \ + --hash=sha256:805cc8de9fd6e7a22da5aed858e0ab16be5a4db6c873dde1d7451c541553aa27 \ + --hash=sha256:8906e71fd8afcb76580404e2a950caef2685df3d2a57fe82a86ac8d33cc007ba \ + --hash=sha256:89f7268c009bc492f506abd6f5265defa7cb3f7487dc21d357c3d290add45082 \ + --hash=sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443 \ + --hash=sha256:8e4549f8a3c6d13d55041925e912bfd834285ef1dd64d6bc7d542583355e2e98 \ + --hash=sha256:8e9afaeb0beff068b4d9cd20d322ba0ee1cecfb0b08db145e4ab4dd44a6b5110 \ + --hash=sha256:98f16a80e917003a12c0580f97b5f875853ebc33e2eaa4bccfc8201ac6869308 \ + --hash=sha256:9e35d3e0144137d9fdae62912e869136164534d64a169f86438bc9561b6ad49f \ + --hash=sha256:9e4424677ce4b47fe73c8b5556d876571f7c6945d264201180db2dc34f676ab5 \ + --hash=sha256:adb6ed2ad29b9e15321d167d152ee909ec73395901b70936f029c3bc6d7f4460 \ + --hash=sha256:aea4f66ff44dfddf8c2cffd66ba6538c5ec67d389285292fe428cb2c738c8aef \ + --hash=sha256:b21041e8cb6a1eb5312dd1d2f80a94d91efffb7a06b70597d44f1bd2dfc315ab \ + --hash=sha256:b2f0073ed0868db1dcd86e052d37279eef185b9c8db5bf61f30f46adac63c909 \ + --hash=sha256:b3a24467af63c67829bfaa61eecf18d5432d4f11992688537be59ecd6ad32f5e \ + --hash=sha256:b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695 \ + --hash=sha256:bba37bc29d4d85761deed3954a1bc62be7cf462b9510b51d367b769a8c8df325 \ + --hash=sha256:bd3a7a9f5847d2fb8c2c6d1c862fa109c31a9abeca1a3c2bd5a64572955b2979 \ + --hash=sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0 \ + --hash=sha256:c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32 \ + --hash=sha256:c3cd545784805de05aafe1dde61752ea49a359ccba9760c1e5d1c88a93bbf2b7 \ + --hash=sha256:c7ac672d699bf36275c035e16b65539931347d68b70667d28984c9fb34e07fa7 \ + --hash=sha256:cb7bbb88aa74908950d979eeaa24dbdf1a865e3c7e45ff0121d8f70387b55f73 \ + --hash=sha256:cd2bd2bbed13e213d6b55dc1d035a4f91748a7d3edc9480c13898b0353708920 \ + --hash=sha256:cda077c2e5b780200b6b3e09d0b42205a3d1c68f30c6dceb90401c13bff8fe74 \ + --hash=sha256:cf28c0c1d4c4bf00f509fa7eb02c58d7caf221b50b467bcb0d9bbf1584d5c821 \ + --hash=sha256:d0d9b7c93578baafcbc5f0b83eaf17b79d345c6f36917ba0c67f45226911d499 \ + --hash=sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000 \ + --hash=sha256:d30291931c915b2ab5717c2974bb95ee891a1cf22ebc16a8006bd59cd210d40a \ + --hash=sha256:d9f64d786b3b1dd742c946c42d15b07497ed14af1a1f3ce840cce27daa0ce913 \ + --hash=sha256:da6cad4e82cb893db4b69105c604d805e0c3ce11501a55b5e9f9083b47d2ffe8 \ + --hash=sha256:df1b10187212b198dd45fa943d8985a3c8cf854aed4923796e0e019e113a1bda \ + --hash=sha256:e04ae107ac591763a47398bb45b568fc38f02dbc4aa44c063f67a131f99346cb \ + --hash=sha256:e6dee3bb76aa4009d5a912180bf5b2de012532998d094acee25d9cb8dee3e44a \ + --hash=sha256:e7e88598032542bd49af7c4747541422884219056c268823ef6e5e89851c8825 \ + --hash=sha256:e98c97502435b53741540a5717a6749ac2ada901056c7db951d33e11c885cc7d \ + --hash=sha256:ec055f6dae239a6299cace477b479cca2fc125c5675482daf1dd886933a1076f \ + --hash=sha256:f74f0f7779cc7ae07d1810aab8ac6b1464c3eafb9e283a40da7309d5e6e48fbb \ + --hash=sha256:fbde1b0c6e81d56f5dccd95dd4a711d9b95df1ae4009a60887e56b27e8d903fa \ + --hash=sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236 \ + --hash=sha256:fd49860271d52127d61197bb50b64f58454e9f578cb4b2c001a6de8b1f50b0b1 + # via apache-beam +oauth2client==4.1.3 \ + --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ + --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 + # via google-apitools +objsize==0.7.1 \ + --hash=sha256:634a0c134c4b1ff2c340fe29caf58bc0a16cb2ff7c556df609d04f026fdf4eca \ + --hash=sha256:91e68d2a3031efb61b0e8cb7f995ddaeb65fe5ace9e737785e029f0932c2e619 + # via apache-beam +opentelemetry-api==1.40.0 \ + --hash=sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f \ + --hash=sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9 + # via + # google-cloud-pubsub + # google-cloud-spanner + # opentelemetry-resourcedetector-gcp + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-resourcedetector-gcp==1.11.0a0 \ + --hash=sha256:5d65a2a039b1d40c6f41421dbb08d5f441368275ac6de6e76a8fccd1f6acb67e \ + --hash=sha256:915a1d6fd15daca9eedd3fc52b0f705375054f2ef140e2e7a6b4cca95a47cdb1 + # via google-cloud-spanner +opentelemetry-sdk==1.40.0 \ + --hash=sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2 \ + --hash=sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1 + # via + # google-cloud-pubsub + # google-cloud-spanner + # opentelemetry-resourcedetector-gcp +opentelemetry-semantic-conventions==0.61b0 \ + --hash=sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a \ + --hash=sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2 + # via + # google-cloud-spanner + # opentelemetry-sdk +orjson==3.11.7 \ + --hash=sha256:043d3006b7d32c7e233b8cfb1f01c651013ea079e08dcef7189a29abd8befe11 \ + --hash=sha256:0527a4510c300e3b406591b0ba69b5dc50031895b0a93743526a3fc45f59d26e \ + --hash=sha256:0724e265bc548af1dedebd9cb3d24b4e1c1e685a343be43e87ba922a5c5fff2f \ + --hash=sha256:136dcd6a2e796dfd9ffca9fc027d778567b0b7c9968d092842d3c323cef88aa8 \ + --hash=sha256:14f440c7268c8f8633d1b3d443a434bd70cb15686117ea6beff8fdc8f5917a1e \ + --hash=sha256:1d98b30cc1313d52d4af17d9c3d307b08389752ec5f2e5febdfada70b0f8c733 \ + --hash=sha256:1eb80451a9c351a71dfaf5b7ccc13ad065405217726b59fdbeadbcc544f9d223 \ + --hash=sha256:1ee5cc7160a821dfe14f130bc8e63e7611051f964b463d9e2a3a573204446a4d \ + --hash=sha256:23d6c20517a97a9daf1d48b580fcdc6f0516c6f4b5038823426033690b4d2650 \ + --hash=sha256:26c3b9132f783b7d7903bf1efb095fed8d4a3a85ec0d334ee8beff3d7a4749d5 \ + --hash=sha256:31c80ce534ac4ea3739c5ee751270646cbc46e45aea7576a38ffec040b4029a1 \ + --hash=sha256:3726be79e36e526e3d9c1aceaadbfb4a04ee80a72ab47b3f3c17fefb9812e7b8 \ + --hash=sha256:390a1dce0c055ddf8adb6aa94a73b45a4a7d7177b5c584b8d1c1947f2ba60fb3 \ + --hash=sha256:3a2479753bbb95b0ebcf7969f562cdb9668e6d12416a35b0dda79febf89cdea2 \ + --hash=sha256:3c4bc6c6ac52cdaa267552544c73e486fecbd710b7ac09bc024d5a78555a22f6 \ + --hash=sha256:411ebaf34d735e25e358a6d9e7978954a9c9d58cfb47bc6683cdc3964cd2f910 \ + --hash=sha256:4682d1db3bcebd2b64757e0ddf9e87ae5f00d29d16c5cdf3a62f561d08cc3dd2 \ + --hash=sha256:4a2e9c5be347b937a2e0203866f12bba36082e89b402ddb9e927d5822e43088d \ + --hash=sha256:57036b27ac8a25d81112eb0cc9835cd4833c5b16e1467816adc0015f59e870dc \ + --hash=sha256:5ede977b5fe5ac91b1dffc0a517ca4542d2ec8a6a4ff7b2652d94f640796342a \ + --hash=sha256:5fdfad2093bdd08245f2e204d977facd5f871c88c4a71230d5bcbd0e43bf6222 \ + --hash=sha256:623ad1b9548ef63886319c16fa317848e465a21513b31a6ad7b57443c3e0dcf5 \ + --hash=sha256:652c6c3af76716f4a9c290371ba2e390ede06f6603edb277b481daf37f6f464e \ + --hash=sha256:6543001328aa857187f905308a028935864aefe9968af3848401b6fe80dbb471 \ + --hash=sha256:6e776b998ac37c0396093d10290e60283f59cfe0fc3fccbd0ccc4bd04dd19892 \ + --hash=sha256:71924496986275a737f38e3f22b4e0878882b3f7a310d2ff4dc96e812789120c \ + --hash=sha256:733ae23ada68b804b222c44affed76b39e30806d38660bf1eb200520d259cc16 \ + --hash=sha256:7477aa6a6ec6139c5cb1cc7b214643592169a5494d200397c7fc95d740d5fcf3 \ + --hash=sha256:79cacb0b52f6004caf92405a7e1f11e6e2de8bdf9019e4f76b44ba045125cd6b \ + --hash=sha256:7ba61079379b0ae29e117db13bda5f28d939766e410d321ec1624afc6a0b0504 \ + --hash=sha256:800988273a014a0541483dc81021247d7eacb0c845a9d1a34a422bc718f41539 \ + --hash=sha256:814be4b49b228cfc0b3c565acf642dd7d13538f966e3ccde61f4f55be3e20785 \ + --hash=sha256:845c3e0d8ded9c9271cd79596b9b552448b885b97110f628fb687aee2eed11c1 \ + --hash=sha256:849e38203e5be40b776ed2718e587faf204d184fc9a008ae441f9442320c0cab \ + --hash=sha256:89e13dd3f89f1c38a9c9eba5fbf7cdc2d1feca82f5f290864b4b7a6aac704576 \ + --hash=sha256:89e440ebc74ce8ab5c7bc4ce6757b4a6b1041becb127df818f6997b5c71aa60b \ + --hash=sha256:8ff206156006da5b847c9304b6308a01e8cdbc8cce824e2779a5ba71c3def141 \ + --hash=sha256:91c81ef070c8f3220054115e1ef468b1c9ce8497b4e526cb9f68ab4dc0a7ac62 \ + --hash=sha256:9487abc2c2086e7c8eb9a211d2ce8855bae0e92586279d0d27b341d5ad76c85c \ + --hash=sha256:962d046ee1765f74a1da723f4b33e3b228fe3a48bd307acce5021dfefe0e29b2 \ + --hash=sha256:996b65230271f1a97026fd0e6a753f51fbc0c335d2ad0c6201f711b0da32693b \ + --hash=sha256:9b1a67243945819ce55d24a30b59d6a168e86220452d2c96f4d1f093e71c0c49 \ + --hash=sha256:9c0b51672e466fd7e56230ffbae7f1639e18d0ce023351fb75da21b71bc2c960 \ + --hash=sha256:9e54f3808e2b6b945078c41aa8d9b5834b28c50843846e97807e5adb75fa9705 \ + --hash=sha256:a02c833f38f36546ba65a452127633afce4cf0dd7296b753d3bb54e55e5c0174 \ + --hash=sha256:a12b80df61aab7b98b490fe9e4879925ba666fccdfcd175252ce4d9035865ace \ + --hash=sha256:a16bcd08ab0bcdfc7e8801d9c4a9cc17e58418e4d48ddc6ded4e9e4b1a94062b \ + --hash=sha256:a56df3239294ea5964adf074c54bcc4f0ccd21636049a2cf3ca9cf03b5d03cf1 \ + --hash=sha256:a709e881723c9b18acddcfb8ba357322491ad553e277cf467e1e7e20e2d90561 \ + --hash=sha256:ab49d4b2a6a1d415ddb9f37a21e02e0d5dbfe10b7870b21bf779fc21e9156157 \ + --hash=sha256:ae9e0b37a834cef7ce8f99de6498f8fad4a2c0bf6bfc3d02abd8ed56aa15b2de \ + --hash=sha256:b4a9eefdc70bf8bf9857f0290f973dec534ac84c35cd6a7f4083be43e7170a8f \ + --hash=sha256:b63c6e6738d7c3470ad01601e23376aa511e50e1f3931395b9f9c722406d1a67 \ + --hash=sha256:b7b1dae39230a393df353827c855a5f176271c23434cfd2db74e0e424e693e10 \ + --hash=sha256:b8d14b71c0b12963fe8a62aac87119f1afdf4cb88a400f61ca5ae581449efcb5 \ + --hash=sha256:b9f95dcdea9d4f805daa9ddf02617a89e484c6985fa03055459f90e87d7a0757 \ + --hash=sha256:b9fc4d0f81f394689e0814617aadc4f2ea0e8025f38c226cbf22d3b5ddbf025d \ + --hash=sha256:bd03ea7606833655048dab1a00734a2875e3e86c276e1d772b2a02556f0d895f \ + --hash=sha256:bd0d68edd7dfca1b2eca9361a44ac9f24b078de3481003159929a0573f21a6bf \ + --hash=sha256:bda117c4148e81f746655d5a3239ae9bd00cb7bc3ca178b5fc5a5997e9744183 \ + --hash=sha256:bf742e149121dc5648ba0a08ea0871e87b660467ef168a3a5e53bc1fbd64bb74 \ + --hash=sha256:c2428d358d85e8da9d37cba18b8c4047c55222007a84f97156a5b22028dfbfc0 \ + --hash=sha256:c2e85fe4698b6a56d5e2ebf7ae87544d668eb6bde1ad1226c13f44663f20ec9e \ + --hash=sha256:c43b8b5bab288b6b90dac410cca7e986a4fa747a2e8f94615aea407da706980d \ + --hash=sha256:cededd6738e1c153530793998e31c05086582b08315db48ab66649768f326baa \ + --hash=sha256:d06e5c5fed5caedd2e540d62e5b1c25e8c82431b9e577c33537e5fa4aa909539 \ + --hash=sha256:d772afdb22555f0c58cfc741bdae44180122b3616faa1ecadb595cd526e4c993 \ + --hash=sha256:d897e81f8d0cbd2abb82226d1860ad2e1ab3ff16d7b08c96ca00df9d45409ef4 \ + --hash=sha256:de0a37f21d0d364954ad5de1970491d7fbd0fb1ef7417d4d56a36dc01ba0c0a0 \ + --hash=sha256:e7745312efa9e11c17fbd3cb3097262d079da26930ae9ae7ba28fb738367cbad \ + --hash=sha256:ed46f17096e28fb28d2975834836a639af7278aa87c84f68ab08fbe5b8bd75fa \ + --hash=sha256:f4f7c956b5215d949a1f65334cf9d7612dde38f20a95f2315deef167def91a6f \ + --hash=sha256:f50979824bde13d32b4320eedd513431c921102796d86be3eee0b58e58a3ecd1 \ + --hash=sha256:f904c24bdeabd4298f7a977ef14ca2a022ca921ed670b92ecd16ab6f3d01f867 + # via apache-beam +overrides==7.7.0 \ + --hash=sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a \ + --hash=sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49 + # via google-cloud-pubsublite +packaging==26.0 \ + --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ + --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 + # via + # apache-beam + # google-cloud-aiplatform + # google-cloud-bigquery +pg8000==1.31.5 \ + --hash=sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201 \ + --hash=sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78 + # via apache-beam +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via keyrings-google-artifactregistry-auth +propcache==0.4.1 \ + --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e \ + --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 \ + --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be \ + --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 \ + --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 \ + --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b \ + --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 \ + --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf \ + --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 \ + --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 \ + --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 \ + --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 \ + --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 \ + --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 \ + --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 \ + --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 \ + --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 \ + --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc \ + --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe \ + --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb \ + --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 \ + --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 \ + --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e \ + --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff \ + --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 \ + --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 \ + --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 \ + --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 \ + --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf \ + --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 \ + --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a \ + --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc \ + --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a \ + --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 \ + --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 \ + --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 \ + --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 \ + --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 \ + --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 \ + --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af \ + --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa \ + --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 \ + --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc \ + --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 \ + --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 \ + --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf \ + --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 \ + --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 \ + --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c \ + --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 \ + --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e \ + --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 \ + --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 \ + --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 \ + --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 \ + --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db \ + --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f \ + --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d \ + --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 \ + --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e \ + --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 \ + --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 \ + --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a \ + --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b \ + --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f \ + --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f \ + --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 \ + --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 \ + --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 \ + --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 \ + --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db \ + --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded \ + --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb \ + --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 \ + --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 \ + --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 \ + --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 \ + --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 \ + --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f \ + --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b \ + --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 \ + --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 \ + --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 \ + --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c \ + --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 \ + --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 \ + --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 \ + --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 \ + --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 \ + --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 \ + --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db \ + --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac \ + --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 \ + --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 \ + --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f \ + --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 \ + --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 \ + --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 \ + --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 \ + --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a \ + --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f \ + --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 \ + --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 \ + --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 \ + --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 \ + --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 \ + --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 \ + --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 \ + --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 \ + --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 \ + --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e \ + --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 \ + --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d \ + --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 \ + --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c \ + --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e \ + --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 \ + --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af \ + --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f \ + --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 \ + --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 \ + --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 + # via + # aiohttp + # yarl +proto-plus==1.27.1 \ + --hash=sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147 \ + --hash=sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc + # via + # apache-beam + # google-api-core + # google-cloud-aiplatform + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-kms + # google-cloud-language + # google-cloud-monitoring + # google-cloud-pubsub + # google-cloud-pubsublite + # google-cloud-recommendations-ai + # google-cloud-resource-manager + # google-cloud-secret-manager + # google-cloud-spanner + # google-cloud-videointelligence + # google-cloud-vision +protobuf==5.29.6 \ + --hash=sha256:36ade6ff88212e91aef4e687a971a11d7d24d6948a66751abc1b3238648f5d05 \ + --hash=sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1 \ + --hash=sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86 \ + --hash=sha256:76e07e6567f8baf827137e8d5b8204b6c7b6488bbbff1bf0a72b383f77999c18 \ + --hash=sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda \ + --hash=sha256:831e2da16b6cc9d8f1654c041dd594eda43391affd3c03a91bea7f7f6da106d6 \ + --hash=sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6 \ + --hash=sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269 \ + --hash=sha256:cb4c86de9cd8a7f3a256b9744220d87b847371c6b2f10bde87768918ef33ba49 \ + --hash=sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723 \ + --hash=sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9 + # via + # apache-beam + # google-api-core + # google-cloud-aiplatform + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-kms + # google-cloud-language + # google-cloud-monitoring + # google-cloud-pubsub + # google-cloud-recommendations-ai + # google-cloud-resource-manager + # google-cloud-secret-manager + # google-cloud-spanner + # google-cloud-videointelligence + # google-cloud-vision + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status + # proto-plus +pyarrow==18.1.0 \ + --hash=sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe \ + --hash=sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e \ + --hash=sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54 \ + --hash=sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99 \ + --hash=sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e \ + --hash=sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9 \ + --hash=sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181 \ + --hash=sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76 \ + --hash=sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c \ + --hash=sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c \ + --hash=sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56 \ + --hash=sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754 \ + --hash=sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b \ + --hash=sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9 \ + --hash=sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992 \ + --hash=sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc \ + --hash=sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7 \ + --hash=sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa \ + --hash=sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b \ + --hash=sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73 \ + --hash=sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812 \ + --hash=sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d \ + --hash=sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052 \ + --hash=sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191 \ + --hash=sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386 \ + --hash=sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324 \ + --hash=sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4 \ + --hash=sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba \ + --hash=sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470 \ + --hash=sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71 \ + --hash=sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30 \ + --hash=sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33 \ + --hash=sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a \ + --hash=sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8 \ + --hash=sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee \ + --hash=sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c \ + --hash=sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6 \ + --hash=sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854 \ + --hash=sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0 \ + --hash=sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21 \ + --hash=sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2 \ + --hash=sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c + # via apache-beam +pyarrow-hotfix==0.7 \ + --hash=sha256:3236f3b5f1260f0e2ac070a55c1a7b339c4bb7267839bd2015e283234e758100 \ + --hash=sha256:59399cd58bdd978b2e42816a4183a55c6472d4e33d183351b6069f11ed42661d + # via apache-beam +pyasn1==0.6.2 \ + --hash=sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf \ + --hash=sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b + # via + # oauth2client + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via + # google-auth + # oauth2client +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.12.5 \ + --hash=sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49 \ + --hash=sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d + # via + # google-cloud-aiplatform + # google-genai +pydantic-core==2.41.5 \ + --hash=sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90 \ + --hash=sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740 \ + --hash=sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504 \ + --hash=sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84 \ + --hash=sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33 \ + --hash=sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c \ + --hash=sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0 \ + --hash=sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e \ + --hash=sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0 \ + --hash=sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a \ + --hash=sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34 \ + --hash=sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2 \ + --hash=sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3 \ + --hash=sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815 \ + --hash=sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14 \ + --hash=sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba \ + --hash=sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375 \ + --hash=sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf \ + --hash=sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963 \ + --hash=sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1 \ + --hash=sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808 \ + --hash=sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553 \ + --hash=sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1 \ + --hash=sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2 \ + --hash=sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5 \ + --hash=sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470 \ + --hash=sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2 \ + --hash=sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b \ + --hash=sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660 \ + --hash=sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c \ + --hash=sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093 \ + --hash=sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5 \ + --hash=sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594 \ + --hash=sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008 \ + --hash=sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a \ + --hash=sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a \ + --hash=sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd \ + --hash=sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284 \ + --hash=sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586 \ + --hash=sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869 \ + --hash=sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294 \ + --hash=sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f \ + --hash=sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66 \ + --hash=sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51 \ + --hash=sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc \ + --hash=sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97 \ + --hash=sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a \ + --hash=sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d \ + --hash=sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9 \ + --hash=sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c \ + --hash=sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07 \ + --hash=sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36 \ + --hash=sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e \ + --hash=sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05 \ + --hash=sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e \ + --hash=sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941 \ + --hash=sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3 \ + --hash=sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612 \ + --hash=sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3 \ + --hash=sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b \ + --hash=sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe \ + --hash=sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146 \ + --hash=sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11 \ + --hash=sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60 \ + --hash=sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd \ + --hash=sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b \ + --hash=sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c \ + --hash=sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a \ + --hash=sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460 \ + --hash=sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1 \ + --hash=sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf \ + --hash=sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf \ + --hash=sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858 \ + --hash=sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2 \ + --hash=sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9 \ + --hash=sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2 \ + --hash=sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3 \ + --hash=sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6 \ + --hash=sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770 \ + --hash=sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d \ + --hash=sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc \ + --hash=sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23 \ + --hash=sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26 \ + --hash=sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa \ + --hash=sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8 \ + --hash=sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d \ + --hash=sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3 \ + --hash=sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d \ + --hash=sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034 \ + --hash=sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9 \ + --hash=sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1 \ + --hash=sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56 \ + --hash=sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b \ + --hash=sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c \ + --hash=sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a \ + --hash=sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e \ + --hash=sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9 \ + --hash=sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5 \ + --hash=sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a \ + --hash=sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556 \ + --hash=sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e \ + --hash=sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49 \ + --hash=sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2 \ + --hash=sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9 \ + --hash=sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b \ + --hash=sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc \ + --hash=sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb \ + --hash=sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0 \ + --hash=sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8 \ + --hash=sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82 \ + --hash=sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69 \ + --hash=sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b \ + --hash=sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c \ + --hash=sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75 \ + --hash=sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5 \ + --hash=sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f \ + --hash=sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad \ + --hash=sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b \ + --hash=sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7 \ + --hash=sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425 \ + --hash=sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52 + # via pydantic +pymongo==4.16.0 \ + --hash=sha256:03f42396c1b2c6f46f5401c5b185adc25f6113716e16d9503977ee5386fca0fb \ + --hash=sha256:12762e7cc0f8374a8cae3b9f9ed8dabb5d438c7b33329232dd9b7de783454033 \ + --hash=sha256:15bb062c0d6d4b0be650410032152de656a2a9a2aa4e1a7443a22695afacb103 \ + --hash=sha256:19a1c96e7f39c7a59a9cfd4d17920cf9382f6f684faeff4649bf587dc59f8edc \ + --hash=sha256:1c01e8a7cd0ea66baf64a118005535ab5bf9f9eb63a1b50ac3935dccf9a54abe \ + --hash=sha256:1d638b0b1b294d95d0fdc73688a3b61e05cc4188872818cd240d51460ccabcb5 \ + --hash=sha256:21d02cc10a158daa20cb040985e280e7e439832fc6b7857bff3d53ef6914ad50 \ + --hash=sha256:2290909275c9b8f637b0a92eb9b89281e18a72922749ebb903403ab6cc7da914 \ + --hash=sha256:25a6b03a68f9907ea6ec8bc7cf4c58a1b51a18e23394f962a6402f8e46d41211 \ + --hash=sha256:2a3ba6be3d8acf64b77cdcd4e36f0e4a8e87965f14a8b09b90ca86f10a1dd2f2 \ + --hash=sha256:2b0714d7764efb29bf9d3c51c964aed7c4c7237b341f9346f15ceaf8321fdb35 \ + --hash=sha256:2cd60cd1e05de7f01927f8e25ca26b3ea2c09de8723241e5d3bcfdc70eaff76b \ + --hash=sha256:2d0082631a7510318befc2b4fdab140481eb4b9dd62d9245e042157085da2a70 \ + --hash=sha256:311d4549d6bf1f8c61d025965aebb5ba29d1481dc6471693ab91610aaffbc0eb \ + --hash=sha256:36ef2fee50eee669587d742fb456e349634b4fcf8926208766078b089054b24b \ + --hash=sha256:3ead8a0050c53eaa55935895d6919d393d0328ec24b2b9115bdbe881aa222673 \ + --hash=sha256:46ffb728d92dd5b09fc034ed91acf5595657c7ca17d4cf3751322cd554153c17 \ + --hash=sha256:4a19ea46a0fe71248965305a020bc076a163311aefbaa1d83e47d06fa30ac747 \ + --hash=sha256:4a9390dce61d705a88218f0d7b54d7e1fa1b421da8129fc7c009e029a9a6b81e \ + --hash=sha256:4c4872299ebe315a79f7f922051061634a64fda95b6b17677ba57ef00b2ba2a4 \ + --hash=sha256:4cd047ba6cc83cc24193b9208c93e134a985ead556183077678c59af7aacc725 \ + --hash=sha256:4d4f7ba040f72a9f43a44059872af5a8c8c660aa5d7f90d5344f2ed1c3c02721 \ + --hash=sha256:4d79aa147ce86aef03079096d83239580006ffb684eead593917186aee407767 \ + --hash=sha256:4fbb8d3552c2ad99d9e236003c0b5f96d5f05e29386ba7abae73949bfebc13dd \ + --hash=sha256:55f8d5a6fe2fa0b823674db2293f92d74cd5f970bc0360f409a1fc21003862d3 \ + --hash=sha256:5b9c6d689bbe5beb156374508133218610e14f8c81e35bc17d7a14e30ab593e6 \ + --hash=sha256:5d9fdb386cf958e6ef6ff537d6149be7edb76c3268cd6833e6c36aa447e4443f \ + --hash=sha256:60307bb91e0ab44e560fe3a211087748b2b5f3e31f403baf41f5b7b0a70bd104 \ + --hash=sha256:61567f712bda04c7545a037e3284b4367cad8d29b3dec84b4bf3b2147020a75b \ + --hash=sha256:66af44ed23686dd5422307619a6db4b56733c5e36fe8c4adf91326dcf993a043 \ + --hash=sha256:6af1aaa26f0835175d2200e62205b78e7ec3ffa430682e322cc91aaa1a0dbf28 \ + --hash=sha256:6b2a20edb5452ac8daa395890eeb076c570790dfce6b7a44d788af74c2f8cf96 \ + --hash=sha256:6f2077ec24e2f1248f9cac7b9a2dfb894e50cc7939fcebfb1759f99304caabef \ + --hash=sha256:77cfd37a43a53b02b7bd930457c7994c924ad8bbe8dff91817904bcbf291b371 \ + --hash=sha256:78037d02389745e247fe5ab0bcad5d1ab30726eaac3ad79219c7d6bbb07eec53 \ + --hash=sha256:7902882ed0efb7f0e991458ab3b8cf0eb052957264949ece2f09b63c58b04f78 \ + --hash=sha256:85dc2f3444c346ea019a371e321ac868a4fab513b7a55fe368f0cc78de8177cc \ + --hash=sha256:8a0f73af1ea56c422b2dcfc0437459148a799ef4231c6aee189d2d4c59d6728f \ + --hash=sha256:8a254d49a9ffe9d7f888e3c677eed3729b14ce85abb08cd74732cead6ccc3c66 \ + --hash=sha256:8ba8405065f6e258a6f872fe62d797a28f383a12178c7153c01ed04e845c600c \ + --hash=sha256:91899dd7fb9a8c50f09c3c1cf0cb73bfbe2737f511f641f19b9650deb61c00ca \ + --hash=sha256:91ac0cb0fe2bf17616c2039dac88d7c9a5088f5cb5829b27c9d250e053664d31 \ + --hash=sha256:92a232af9927710de08a6c16a9710cc1b175fb9179c0d946cd4e213b92b2a69a \ + --hash=sha256:948152b30eddeae8355495f9943a3bf66b708295c0b9b6f467de1c620f215487 \ + --hash=sha256:96aa7ab896889bf330209d26459e493d00f8855772a9453bfb4520bb1f495baf \ + --hash=sha256:9caacac0dd105e2555521002e2d17afc08665187017b466b5753e84c016628e6 \ + --hash=sha256:9d9885aad05f82fd7ea0c9ca505d60939746b39263fa273d0125170da8f59098 \ + --hash=sha256:9dc2c00bed568732b89e211b6adca389053d5e6d2d5a8979e80b813c3ec4d1f9 \ + --hash=sha256:a1bf44e13cf2d44d2ea2e928a8140d5d667304abe1a61c4d55b4906f389fbe64 \ + --hash=sha256:aa30cd16ddd2f216d07ba01d9635c873e97ddb041c61cf0847254edc37d1c60e \ + --hash=sha256:acda193f440dd88c2023cb00aa8bd7b93a9df59978306d14d87a8b12fe426b05 \ + --hash=sha256:bd4911c40a43a821dfd93038ac824b756b6e703e26e951718522d29f6eb166a8 \ + --hash=sha256:be1099a8295b1a722d03fb7b48be895d30f4301419a583dcf50e9045968a041c \ + --hash=sha256:c126fb72be2518395cc0465d4bae03125119136462e1945aea19840e45d89cfc \ + --hash=sha256:c53338613043038005bf2e41a2fafa08d29cdbc0ce80891b5366c819456c1ae9 \ + --hash=sha256:c789236366525c3ee3cd6e4e450a9ff629a7d1f4d88b8e18a0aea0615fd7ecf8 \ + --hash=sha256:cf0ec79e8ca7077f455d14d915d629385153b6a11abc0b93283ed73a8013e376 \ + --hash=sha256:d15f060bc6d0964a8bb70aba8f0cb6d11ae99715438f640cff11bbcf172eb0e8 \ + --hash=sha256:d284bf68daffc57516535f752e290609b3b643f4bd54b28fc13cb16a89a8bda6 \ + --hash=sha256:dabbf3c14de75a20cc3c30bf0c6527157224a93dfb605838eabb1a2ee3be008d \ + --hash=sha256:dbbc5b254c36c37d10abb50e899bc3939bbb7ab1e7c659614409af99bd3e7675 \ + --hash=sha256:dfc320f08ea9a7ec5b2403dc4e8150636f0d6150f4b9792faaae539c88e7db3b \ + --hash=sha256:e2d509786344aa844ae243f68f833ca1ac92ac3e35a92ae038e2ceb44aa355ef \ + --hash=sha256:e37469602473f41221cea93fd3736708f561f0fa08ab6b2873dd962014390d52 \ + --hash=sha256:ed162b2227f98d5b270ecbe1d53be56c8c81db08a1a8f5f02d89c7bb4d19591d \ + --hash=sha256:efe020c46ce3c3a89af6baec6569635812129df6fb6cf76d4943af3ba6ee2069 \ + --hash=sha256:f1c5f1f818b669875d191323a48912d3fcd2e4906410e8297bb09ac50c4d5ccc \ + --hash=sha256:f25001a955073b80510c0c3db0e043dbbc36904fd69e511c74e3d8640b8a5111 \ + --hash=sha256:f3867dc225d9423c245a51eaac2cfcd53dde8e0a8d8090bb6aed6e31bd6c2d4f \ + --hash=sha256:f513b2c6c0d5c491f478422f6b5b5c27ac1af06a54c93ef8631806f7231bd92e \ + --hash=sha256:f6e42c1bc985d9beee884780ae6048790eb4cd565c46251932906bdb1630034a + # via apache-beam +pymysql==1.1.2 \ + --hash=sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03 \ + --hash=sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9 + # via apache-beam +pyparsing==3.3.2 \ + --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ + --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc + # via httplib2 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via + # apache-beam + # betterproto + # google-cloud-bigquery + # pg8000 +python-tds==1.17.1 \ + --hash=sha256:35cb210b1a54e5ccc91570a83d4e9a2a16682cbeb00bede06fd6cdf9afa9762f \ + --hash=sha256:c97483a9adf1dcab8bee66e83429acc502753f389d134553edd818348b94ced0 + # via apache-beam +pytz==2026.1.post1 \ + --hash=sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1 \ + --hash=sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a + # via apache-beam +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via apache-beam +regex==2026.2.28 \ + --hash=sha256:00945d007fd74a9084d2ab79b695b595c6b7ba3698972fadd43e23230c6979c1 \ + --hash=sha256:00f2b8d9615aa165fdff0a13f1a92049bfad555ee91e20d246a51aa0b556c60a \ + --hash=sha256:01d65fd24206c8e1e97e2e31b286c59009636c022eb5d003f52760b0f42155d4 \ + --hash=sha256:02473c954af35dd2defeb07e44182f5705b30ea3f351a7cbffa9177beb14da5d \ + --hash=sha256:03a83cc26aa2acda6b8b9dfe748cf9e84cbd390c424a1de34fdcef58961a297a \ + --hash=sha256:09500be324f49b470d907b3ef8af9afe857f5cca486f853853f7945ddbf75911 \ + --hash=sha256:0b1d2b07614d95fa2bf8a63fd1e98bd8fa2b4848dc91b1efbc8ba219fdd73952 \ + --hash=sha256:0d25a10811de831c2baa6aef3c0be91622f44dd8d31dd12e69f6398efb15e48b \ + --hash=sha256:0d5bef2031cbf38757a0b0bc4298bb4824b6332d28edc16b39247228fbdbad97 \ + --hash=sha256:10d28e19bd4888e4abf43bd3925f3c134c52fdf7259219003588a42e24c2aa25 \ + --hash=sha256:180e08a435a0319e6a4821c3468da18dc7001987e1c17ae1335488dfe7518dd8 \ + --hash=sha256:195237dc327858a7721bf8b0bbbef797554bc13563c3591e91cd0767bacbe359 \ + --hash=sha256:19a9c9e0a8f24f39d575a6a854d516b48ffe4cbdcb9de55cb0570a032556ecff \ + --hash=sha256:1c2c95e1a2b0f89d01e821ff4de1be4b5d73d1f4b0bf679fa27c1ad8d2327f1a \ + --hash=sha256:1d367257cd86c1cbb97ea94e77b373a0bbc2224976e247f173d19e8f18b4afa7 \ + --hash=sha256:1e496956106fd59ba6322a8ea17141a27c5040e5ee8f9433ae92d4e5204462a0 \ + --hash=sha256:1f8b17be5c27a684ea6759983c13506bd77bfc7c0347dff41b18ce5ddd2ee09a \ + --hash=sha256:2234059cfe33d9813a3677ef7667999caea9eeaa83fef98eb6ce15c6cf9e0215 \ + --hash=sha256:25b6eb660c5cf4b8c3407a1ed462abba26a926cc9965e164268a3267bcc06a43 \ + --hash=sha256:2954379dd20752e82d22accf3ff465311cbb2bac6c1f92c4afd400e1757f7451 \ + --hash=sha256:2afa673660928d0b63d84353c6c08a8a476ddfc4a47e11742949d182e6863ce8 \ + --hash=sha256:2b2b23587b26496ff5fd40df4278becdf386813ec00dc3533fa43a4cf0e2ad3c \ + --hash=sha256:2fb950ac1d88e6b6a9414381f403797b236f9fa17e1eee07683af72b1634207b \ + --hash=sha256:3935174fa4d9f70525a4367aaff3cb8bc0548129d114260c29d9dfa4a5b41692 \ + --hash=sha256:39bb5727650b9a0275c6a6690f9bb3fe693a7e6cc5c3155b1240aedf8926423e \ + --hash=sha256:3b24bd7e9d85dc7c6a8bd2aa14ecd234274a0248335a02adeb25448aecdd420d \ + --hash=sha256:4390c365fd2d45278f45afd4673cb90f7285f5701607e3ad4274df08e36140ae \ + --hash=sha256:481df4623fa4969c8b11f3433ed7d5e3dc9cec0f008356c3212b3933fb77e3d8 \ + --hash=sha256:4f5c0b182ad4269e7381b7c27fdb0408399881f7a92a4624fd5487f2971dfc11 \ + --hash=sha256:50c2fc924749543e0eacc93ada6aeeb3ea5f6715825624baa0dccaec771668ae \ + --hash=sha256:511f7419f7afab475fd4d639d4aedfc54205bcb0800066753ef68a59f0f330b5 \ + --hash=sha256:516604edd17b1c2c3e579cf4e9b25a53bf8fa6e7cedddf1127804d3e0140ca64 \ + --hash=sha256:52b017b35ac2214d0db5f4f90e303634dc44e4aba4bd6235a27f97ecbe5b0472 \ + --hash=sha256:5a932ea8ad5d0430351ff9c76c8db34db0d9f53c1d78f06022a21f4e290c5c18 \ + --hash=sha256:5cdcc17d935c8f9d3f4db5c2ebe2640c332e3822ad5d23c2f8e0228e6947943a \ + --hash=sha256:5d10303dd18cedfd4d095543998404df656088240bcfd3cd20a8f95b861f74bd \ + --hash=sha256:5e68192bb3a1d6fb2836da24aa494e413ea65853a21505e142e5b1064a595f3d \ + --hash=sha256:64e7c6ad614573e0640f271e811a408d79a9e1fe62a46adb602f598df42a818d \ + --hash=sha256:6591f281cb44dc13de9585b552cec6fc6cf47fb2fe7a48892295ee9bc4a612f9 \ + --hash=sha256:69fc560ccbf08a09dc9b52ab69cacfae51e0ed80dc5693078bdc97db2f91ae96 \ + --hash=sha256:6d63a07e5ec8ce7184452cb00c41c37b49e67dc4f73b2955b5b8e782ea970784 \ + --hash=sha256:6db7bfae0f8a2793ff1f7021468ea55e2699d0790eb58ee6ab36ae43aa00bc5b \ + --hash=sha256:71a911098be38c859ceb3f9a9ce43f4ed9f4c6720ad8684a066ea246b76ad9ff \ + --hash=sha256:73cdcdbba8028167ea81490c7f45280113e41db2c7afb65a276f4711fa3bcbff \ + --hash=sha256:78454178c7df31372ea737996fb7f36b3c2c92cccc641d251e072478afb4babc \ + --hash=sha256:7900157786428a79615a8264dac1f12c9b02957c473c8110c6b1f972dcecaddf \ + --hash=sha256:7ab218076eb0944549e7fe74cf0e2b83a82edb27e81cc87411f76240865e04d5 \ + --hash=sha256:7c1b34dfa72f826f535b20712afa9bb3ba580020e834f3c69866c5bddbf10098 \ + --hash=sha256:851fa70df44325e1e4cdb79c5e676e91a78147b1b543db2aec8734d2add30ec2 \ + --hash=sha256:864cdd1a2ef5716b0ab468af40139e62ede1b3a53386b375ec0786bb6783fc05 \ + --hash=sha256:8710d61737b0c0ce6836b1da7109f20d495e49b3809f30e27e9560be67a257bf \ + --hash=sha256:9036b400b20e4858d56d117108d7813ed07bb7803e3eed766675862131135ca6 \ + --hash=sha256:9185cc63359862a6e80fe97f696e04b0ad9a11c4ac0a4a927f979f611bfe3768 \ + --hash=sha256:948c12ef30ecedb128903c2c2678b339746eb7c689c5c21957c4a23950c96d15 \ + --hash=sha256:94d63db12e45a9b9f064bfe4800cefefc7e5f182052e4c1b774d46a40ab1d9bb \ + --hash=sha256:96f6269a2882fbb0ee76967116b83679dc628e68eaea44e90884b8d53d833881 \ + --hash=sha256:97054c55db06ab020342cc0d35d6f62a465fa7662871190175f1ad6c655c028f \ + --hash=sha256:98adf340100cbe6fbaf8e6dc75e28f2c191b1be50ffefe292fb0e6f6eefdb0d8 \ + --hash=sha256:99985a2c277dcb9ccb63f937451af5d65177af1efdeb8173ac55b61095a0a05c \ + --hash=sha256:9b65d33a17101569f86d9c5966a8b1d7fbf8afdda5a8aa219301b0a80f58cf7d \ + --hash=sha256:9dd450db6458387167e033cfa80887a34c99c81d26da1bf8b0b41bf8c9cac88e \ + --hash=sha256:a25c7701e4f7a70021db9aaf4a4a0a67033c6318752146e03d1b94d32006217e \ + --hash=sha256:a448af01e3d8031c89c5d902040b124a5e921a25c4e5e07a861ca591ce429341 \ + --hash=sha256:a5dac14d0872eeb35260a8e30bac07ddf22adc1e3a0635b52b02e180d17c9c7e \ + --hash=sha256:a729e47d418ea11d03469f321aaf67cdee8954cde3ff2cf8403ab87951ad10f2 \ + --hash=sha256:aaffaecffcd2479ce87aa1e74076c221700b7c804e48e98e62500ee748f0f550 \ + --hash=sha256:b059e71ec363968671693a78c5053bd9cb2fe410f9b8e4657e88377ebd603a2e \ + --hash=sha256:b387a0d092dac157fb026d737dde35ff3e49ef27f285343e7c6401851239df27 \ + --hash=sha256:b389c61aa28a79c2e0527ac36da579869c2e235a5b208a12c5b5318cda2501d8 \ + --hash=sha256:b42f7466e32bf15a961cf09f35fa6323cc72e64d3d2c990b10de1274a5da0a59 \ + --hash=sha256:b49eb78048c6354f49e91e4b77da21257fecb92256b6d599ae44403cab30b05b \ + --hash=sha256:b5acd4b6a95f37c3c3828e5d053a7d4edaedb85de551db0153754924cb7c83e3 \ + --hash=sha256:b8b3f1be1738feadc69f62daa250c933e85c6f34fa378f54a7ff43807c1b9117 \ + --hash=sha256:b8cf76f1a29f0e99dcfd7aef1551a9827588aae5a737fe31442021165f1920dc \ + --hash=sha256:ba55c50f408fb5c346a3a02d2ce0ebc839784e24f7c9684fde328ff063c3cdea \ + --hash=sha256:bba2b18d70eeb7b79950f12f633beeecd923f7c9ad6f6bae28e59b4cb3ab046b \ + --hash=sha256:bbb882061f742eb5d46f2f1bd5304055be0a66b783576de3d7eef1bed4778a6e \ + --hash=sha256:bcb399ed84eabf4282587ba151f2732ad8168e66f1d3f85b1d038868fe547703 \ + --hash=sha256:bd477d5f79920338107f04aa645f094032d9e3030cc55be581df3d1ef61aa318 \ + --hash=sha256:bec23c11cbbf09a4df32fe50d57cbdd777bc442269b6e39a1775654f1c95dee2 \ + --hash=sha256:c0b5ccbb8ffb433939d248707d4a8b31993cb76ab1a0187ca886bf50e96df952 \ + --hash=sha256:c15af43c72a7fb0c97cbc66fa36a43546eddc5c06a662b64a0cbf30d6ac40944 \ + --hash=sha256:c7815afb0ca45456613fdaf60ea9c993715511c8d53a83bc468305cbc0ee23c7 \ + --hash=sha256:cb3b1db8ff6c7b8bf838ab05583ea15230cb2f678e569ab0e3a24d1e8320940b \ + --hash=sha256:d0b02e8b7e5874b48ae0f077ecca61c1a6a9f9895e9c6dfb191b55b242862033 \ + --hash=sha256:d6b08a06976ff4fb0d83077022fde3eca06c55432bb997d8c0495b9a4e9872f4 \ + --hash=sha256:d6cfe798d8da41bb1862ed6e0cba14003d387c3c0c4a5d45591076ae9f0ce2f8 \ + --hash=sha256:d8511a01d0e4ee1992eb3ba19e09bc1866fe03f05129c3aec3fdc4cbc77aad3f \ + --hash=sha256:dc8ed8c3f41c27acb83f7b6a9eb727a73fc6663441890c5cb3426a5f6a91ce7d \ + --hash=sha256:dd8847c4978bc3c7e6c826fb745f5570e518b8459ac2892151ce6627c7bc00d5 \ + --hash=sha256:de0cf053139f96219ccfabb4a8dd2d217c8c82cb206c91d9f109f3f552d6b43d \ + --hash=sha256:dee50f1be42222f89767b64b283283ef963189da0dda4a515aa54a5563c62dec \ + --hash=sha256:e1e7b24cb3ae9953a560c563045d1ba56ee4749fbd05cf21ba571069bd7be81b \ + --hash=sha256:e59bc8f30414d283ae8ee1617b13d8112e7135cb92830f0ec3688cb29152585a \ + --hash=sha256:e61eea47230eba62a31f3e8a0e3164d0f37ef9f40529fb2c79361bc6b53d2a92 \ + --hash=sha256:e621fb7c8dc147419b28e1702f58a0177ff8308a76fa295c71f3e7827849f5d9 \ + --hash=sha256:e71dcecaa113eebcc96622c17692672c2d104b1d71ddf7adeda90da7ddeb26fc \ + --hash=sha256:e7ce83654d1ab701cb619285a18a8e5a889c1216d746ddc710c914ca5fd71022 \ + --hash=sha256:e8c8cb2deba42f5ec1ede46374e990f8adc5e6456a57ac1a261b19be6f28e4e6 \ + --hash=sha256:ec0c608b7a7465ffadb344ed7c987ff2f11ee03f6a130b569aa74d8a70e8333c \ + --hash=sha256:ec6f5674c5dc836994f50f1186dd1fafde4be0666aae201ae2fcc3d29d8adf27 \ + --hash=sha256:edb1b1b3a5576c56f08ac46f108c40333f222ebfd5cf63afdfa3aab0791ebe5b \ + --hash=sha256:ef77bdde9c9eba3f7fa5b58084b29bbcc74bcf55fdbeaa67c102a35b5bd7e7cc \ + --hash=sha256:f2791948f7c70bb9335a9102df45e93d428f4b8128020d85920223925d73b9e1 \ + --hash=sha256:f467cb602f03fbd1ab1908f68b53c649ce393fde056628dc8c7e634dab6bfc07 \ + --hash=sha256:f8ed9a5d4612df9d4de15878f0bc6aa7a268afbe5af21a3fdd97fa19516e978c \ + --hash=sha256:fa539be029844c0ce1114762d2952ab6cfdd7c7c9bd72e0db26b94c3c36dcc5a \ + --hash=sha256:fb1c4ff62277d87a7335f2c1ea4e0387b8f2b3ad88a64efd9943906aafad4f33 \ + --hash=sha256:fb4db2f17e6484904f986c5a657cec85574c76b5c5e61c7aae9ffa1bc6224f95 \ + --hash=sha256:fb66e5245db9652abd7196ace599b04d9c0e4aa7c8f0e2803938377835780081 \ + --hash=sha256:fc48c500838be6882b32748f60a15229d2dea96e59ef341eaa96ec83538f498d \ + --hash=sha256:fcf26c3c6d0da98fada8ae4ef0aa1c3405a431c0a77eb17306d38a89b02adcd7 \ + --hash=sha256:fd0ce43e71d825b7c0661f9c54d4d74bd97c56c3fd102a8985bcfea48236bacb \ + --hash=sha256:fd63453f10d29097cc3dc62d070746523973fb5aa1c66d25f8558bebd47fed61 + # via apache-beam +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf + # via + # apache-beam + # cloud-sql-python-connector + # google-api-core + # google-auth + # google-cloud-bigquery + # google-cloud-storage + # google-genai + # keyrings-google-artifactregistry-auth + # opentelemetry-resourcedetector-gcp +rsa==4.9.1 \ + --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ + --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 + # via + # google-auth + # oauth2client +scramp==1.4.8 \ + --hash=sha256:87c2f15976845a2872fe5490a06097f0d01813cceb53774ea168c911f2ad025c \ + --hash=sha256:bd018fabfe46343cceeb9f1c3e8d23f55770271e777e3accbfaee3ff0a316e71 + # via pg8000 +secretstorage==3.5.0 \ + --hash=sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137 \ + --hash=sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be + # via keyring +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # google-apitools + # oauth2client + # python-dateutil +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via google-genai +sortedcontainers==2.4.0 \ + --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \ + --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 + # via apache-beam +sqlparse==0.5.5 \ + --hash=sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba \ + --hash=sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e + # via google-cloud-spanner +tenacity==9.1.4 \ + --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 \ + --hash=sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a + # via google-genai +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via + # aiosignal + # anyio + # apache-beam + # betterproto + # google-cloud-aiplatform + # google-genai + # opentelemetry-api + # opentelemetry-resourcedetector-gcp + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 + # via pydantic +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 + # via requests +websockets==16.0 \ + --hash=sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c \ + --hash=sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a \ + --hash=sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe \ + --hash=sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e \ + --hash=sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec \ + --hash=sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1 \ + --hash=sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64 \ + --hash=sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3 \ + --hash=sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8 \ + --hash=sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206 \ + --hash=sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3 \ + --hash=sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156 \ + --hash=sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d \ + --hash=sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9 \ + --hash=sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad \ + --hash=sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2 \ + --hash=sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03 \ + --hash=sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8 \ + --hash=sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230 \ + --hash=sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8 \ + --hash=sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea \ + --hash=sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641 \ + --hash=sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957 \ + --hash=sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6 \ + --hash=sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6 \ + --hash=sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5 \ + --hash=sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f \ + --hash=sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00 \ + --hash=sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e \ + --hash=sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b \ + --hash=sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72 \ + --hash=sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39 \ + --hash=sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9 \ + --hash=sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79 \ + --hash=sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0 \ + --hash=sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac \ + --hash=sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35 \ + --hash=sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0 \ + --hash=sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5 \ + --hash=sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c \ + --hash=sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8 \ + --hash=sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1 \ + --hash=sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244 \ + --hash=sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3 \ + --hash=sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767 \ + --hash=sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a \ + --hash=sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d \ + --hash=sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd \ + --hash=sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e \ + --hash=sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944 \ + --hash=sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82 \ + --hash=sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d \ + --hash=sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4 \ + --hash=sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5 \ + --hash=sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904 \ + --hash=sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde \ + --hash=sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f \ + --hash=sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c \ + --hash=sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89 \ + --hash=sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da \ + --hash=sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4 + # via google-genai +yarl==1.23.0 \ + --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \ + --hash=sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4 \ + --hash=sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85 \ + --hash=sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993 \ + --hash=sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222 \ + --hash=sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de \ + --hash=sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25 \ + --hash=sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e \ + --hash=sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2 \ + --hash=sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e \ + --hash=sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860 \ + --hash=sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957 \ + --hash=sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760 \ + --hash=sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52 \ + --hash=sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788 \ + --hash=sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912 \ + --hash=sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719 \ + --hash=sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035 \ + --hash=sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220 \ + --hash=sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412 \ + --hash=sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05 \ + --hash=sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41 \ + --hash=sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4 \ + --hash=sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4 \ + --hash=sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd \ + --hash=sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748 \ + --hash=sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a \ + --hash=sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4 \ + --hash=sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34 \ + --hash=sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069 \ + --hash=sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25 \ + --hash=sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2 \ + --hash=sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb \ + --hash=sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f \ + --hash=sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5 \ + --hash=sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8 \ + --hash=sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c \ + --hash=sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512 \ + --hash=sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6 \ + --hash=sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5 \ + --hash=sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9 \ + --hash=sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072 \ + --hash=sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5 \ + --hash=sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277 \ + --hash=sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a \ + --hash=sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6 \ + --hash=sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae \ + --hash=sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26 \ + --hash=sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2 \ + --hash=sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4 \ + --hash=sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70 \ + --hash=sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723 \ + --hash=sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c \ + --hash=sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9 \ + --hash=sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5 \ + --hash=sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e \ + --hash=sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c \ + --hash=sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4 \ + --hash=sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0 \ + --hash=sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2 \ + --hash=sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b \ + --hash=sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7 \ + --hash=sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750 \ + --hash=sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2 \ + --hash=sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474 \ + --hash=sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716 \ + --hash=sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7 \ + --hash=sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123 \ + --hash=sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007 \ + --hash=sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595 \ + --hash=sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe \ + --hash=sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea \ + --hash=sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598 \ + --hash=sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679 \ + --hash=sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8 \ + --hash=sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83 \ + --hash=sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6 \ + --hash=sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f \ + --hash=sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94 \ + --hash=sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51 \ + --hash=sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120 \ + --hash=sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039 \ + --hash=sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1 \ + --hash=sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05 \ + --hash=sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb \ + --hash=sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144 \ + --hash=sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa \ + --hash=sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a \ + --hash=sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99 \ + --hash=sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928 \ + --hash=sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d \ + --hash=sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3 \ + --hash=sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434 \ + --hash=sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86 \ + --hash=sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46 \ + --hash=sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319 \ + --hash=sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67 \ + --hash=sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c \ + --hash=sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169 \ + --hash=sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c \ + --hash=sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59 \ + --hash=sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107 \ + --hash=sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4 \ + --hash=sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a \ + --hash=sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb \ + --hash=sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f \ + --hash=sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769 \ + --hash=sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432 \ + --hash=sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090 \ + --hash=sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764 \ + --hash=sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d \ + --hash=sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4 \ + --hash=sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b \ + --hash=sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d \ + --hash=sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543 \ + --hash=sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24 \ + --hash=sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5 \ + --hash=sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b \ + --hash=sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d \ + --hash=sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b \ + --hash=sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6 \ + --hash=sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735 \ + --hash=sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e \ + --hash=sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28 \ + --hash=sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3 \ + --hash=sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401 \ + --hash=sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6 \ + --hash=sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d + # via aiohttp +zipp==3.23.0 \ + --hash=sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e \ + --hash=sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166 + # via importlib-metadata +zstandard==0.25.0 \ + --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ + --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ + --hash=sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3 \ + --hash=sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f \ + --hash=sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6 \ + --hash=sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936 \ + --hash=sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431 \ + --hash=sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250 \ + --hash=sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa \ + --hash=sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f \ + --hash=sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851 \ + --hash=sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3 \ + --hash=sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9 \ + --hash=sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6 \ + --hash=sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362 \ + --hash=sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649 \ + --hash=sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb \ + --hash=sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5 \ + --hash=sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439 \ + --hash=sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137 \ + --hash=sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa \ + --hash=sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd \ + --hash=sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701 \ + --hash=sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0 \ + --hash=sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043 \ + --hash=sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1 \ + --hash=sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860 \ + --hash=sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611 \ + --hash=sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53 \ + --hash=sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b \ + --hash=sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088 \ + --hash=sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e \ + --hash=sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa \ + --hash=sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2 \ + --hash=sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0 \ + --hash=sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7 \ + --hash=sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf \ + --hash=sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388 \ + --hash=sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530 \ + --hash=sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577 \ + --hash=sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902 \ + --hash=sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc \ + --hash=sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98 \ + --hash=sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a \ + --hash=sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097 \ + --hash=sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea \ + --hash=sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09 \ + --hash=sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb \ + --hash=sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7 \ + --hash=sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74 \ + --hash=sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b \ + --hash=sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b \ + --hash=sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b \ + --hash=sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91 \ + --hash=sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150 \ + --hash=sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049 \ + --hash=sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27 \ + --hash=sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a \ + --hash=sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00 \ + --hash=sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd \ + --hash=sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072 \ + --hash=sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c \ + --hash=sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c \ + --hash=sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065 \ + --hash=sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512 \ + --hash=sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1 \ + --hash=sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f \ + --hash=sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2 \ + --hash=sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df \ + --hash=sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab \ + --hash=sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7 \ + --hash=sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b \ + --hash=sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550 \ + --hash=sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0 \ + --hash=sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea \ + --hash=sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277 \ + --hash=sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2 \ + --hash=sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7 \ + --hash=sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778 \ + --hash=sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859 \ + --hash=sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d \ + --hash=sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751 \ + --hash=sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12 \ + --hash=sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2 \ + --hash=sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d \ + --hash=sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0 \ + --hash=sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3 \ + --hash=sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd \ + --hash=sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e \ + --hash=sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f \ + --hash=sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e \ + --hash=sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94 \ + --hash=sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708 \ + --hash=sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313 \ + --hash=sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4 \ + --hash=sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c \ + --hash=sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344 \ + --hash=sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551 \ + --hash=sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01 + # via apache-beam + +# The following packages are considered to be unsafe in a requirements file: +setuptools==82.0.0 \ + --hash=sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb \ + --hash=sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0 + # via -r python/default_base_bqmonitor_requirements.txt + diff --git a/python/src/main/python/bigquery-anomaly-detection/setup.py b/python/src/main/python/bigquery-anomaly-detection/setup.py new file mode 100644 index 0000000000..3e9ddf3df1 --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/setup.py @@ -0,0 +1,14 @@ +from setuptools import setup, find_packages + +setup( + name='bqmonitor', + version='0.1.0', + description='BigQuery anomaly monitoring pipeline (Dataflow Flex Template)', + package_dir={'': 'src'}, + packages=find_packages(where='src'), + python_requires='>=3.11', + install_requires=[ + 'apache-beam[gcp]>=2.71.0', + 'google-cloud-bigquery-storage', + ], +) diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/PKG-INFO b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/PKG-INFO new file mode 100644 index 0000000000..447713edad --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/PKG-INFO @@ -0,0 +1,8 @@ +Metadata-Version: 2.4 +Name: bqmonitor +Version: 0.1.0 +Summary: BigQuery anomaly monitoring pipeline (Dataflow Flex Template) +Requires-Python: >=3.11 +Requires-Dist: apache-beam[gcp]==2.71.0 +Requires-Dist: google-cloud-bigquery-storage +Dynamic: requires-python diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/SOURCES.txt b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/SOURCES.txt new file mode 100644 index 0000000000..8a662aa9f5 --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/SOURCES.txt @@ -0,0 +1,12 @@ +pyproject.toml +setup.py +src/bqmonitor/__init__.py +src/bqmonitor/cdc.py +src/bqmonitor/metric.py +src/bqmonitor/pipeline.py +src/bqmonitor/safe_eval.py +src/bqmonitor.egg-info/PKG-INFO +src/bqmonitor.egg-info/SOURCES.txt +src/bqmonitor.egg-info/dependency_links.txt +src/bqmonitor.egg-info/requires.txt +src/bqmonitor.egg-info/top_level.txt \ No newline at end of file diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/dependency_links.txt b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/dependency_links.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/requires.txt b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/requires.txt new file mode 100644 index 0000000000..302e286e7f --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/requires.txt @@ -0,0 +1,2 @@ +apache-beam[gcp]==2.71.0 +google-cloud-bigquery-storage diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/top_level.txt b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/top_level.txt new file mode 100644 index 0000000000..5930473c1e --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor.egg-info/top_level.txt @@ -0,0 +1 @@ +bqmonitor diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/__init__.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py new file mode 100644 index 0000000000..0bb98127a8 --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/cdc.py @@ -0,0 +1,1402 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Streaming source for BigQuery change history (APPENDS/CHANGES functions). + +This module provides ``ReadBigQueryChangeHistory``, a streaming PTransform +that continuously polls BigQuery APPENDS() or CHANGES() functions and emits +changed rows as an unbounded PCollection. + +**Status: Experimental**: API may change without notice. + +Usage:: + + import apache_beam as beam + from bqmonitor.cdc import ReadBigQueryChangeHistory + + with beam.Pipeline(options=pipeline_options) as p: + changes = ( + p + | ReadBigQueryChangeHistory( + table='my-project:my_dataset.my_table', + change_function='APPENDS', + poll_interval_sec=60)) + +Architecture: + Poll: Polling SDF emits lightweight _QueryRange instructions. + Query: _ExecuteQueryFn runs the BQ query, writes to a temp table. + Read: SDF reads temp table via Storage Read API with dynamic splitting. + Cleanup: Stateful DoFn tracks stream completion, deletes temp tables. +""" + +import dataclasses +import datetime +import logging +import random +import sys +import time +import uuid +from typing import Any +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Tuple + +import apache_beam as beam +from apache_beam.io.gcp import bigquery_tools +from apache_beam.io.gcp.internal.clients import bigquery +from apache_beam.io.iobase import WatermarkEstimator +from apache_beam.io.restriction_trackers import OffsetRange +from apache_beam.io.restriction_trackers import OffsetRestrictionTracker +from apache_beam.io.watermark_estimators import ManualWatermarkEstimator +from apache_beam.metrics import Metrics +from apache_beam.transforms.core import WatermarkEstimatorProvider +from apache_beam.transforms import trigger as beam_trigger +from apache_beam.transforms.window import GlobalWindows +from apache_beam.transforms.window import TimestampedValue +from apache_beam.utils import retry +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import Duration +from apache_beam.utils.timestamp import Timestamp + +try: + from apitools.base.py.exceptions import HttpError +except ImportError: + HttpError = None # type: ignore + +try: + from google.cloud import bigquery_storage_v1 as bq_storage +except ImportError: + bq_storage = None # type: ignore + +try: + import pyarrow +except ImportError: + pyarrow = None # type: ignore + +_LOGGER = logging.getLogger(__name__) + +__all__ = ['ReadBigQueryChangeHistory'] + +# Max time range for CHANGES() queries: 1 day. +_MAX_CHANGES_RANGE = Duration(seconds=86400) + +# Side output tag for cleanup signals between the Read SDF and Cleanup DoFn. +_CLEANUP_TAG = 'cleanup' + +# Default number of Storage Read API streams to request. +# Matches ReadFromBigQuery's MIN_SPLIT_COUNT to enable parallelism. +# The server may return fewer streams if the table is small. +_DEFAULT_MAX_STREAMS = 10 + +# Default table expiration for auto-created temp datasets: 24 hours in ms. +# Tables created in the dataset auto-expire after this duration if not +# explicitly deleted, acting as a safety net for orphaned temp tables +# (e.g. pipeline crash before cleanup runs). +_DEFAULT_TABLE_EXPIRATION_MS = 24 * 60 * 60 * 1000 + + +@dataclasses.dataclass +class _QueryResult: + """Bridges the Query step (query execution) to the Read SDF. + + After _ExecuteQueryFn runs a CHANGES/APPENDS query, it emits a _QueryResult + pointing to the temp table containing query results. The Read SDF reads + rows from that temp table via the Storage Read API. + + range_start/range_end define the time window this query covers as Beam + Timestamps (int microseconds internally). The Read SDF uses range_start + to set an initial watermark hold so the runner doesn't advance the + watermark past the data's timestamps. + """ + temp_table_ref: 'bigquery.TableReference' + range_start: Timestamp + range_end: Timestamp + + +@dataclasses.dataclass +class _PollConfig: + """Input element for the polling SDF. + + Only contains start_time (Beam Timestamp), which + _PollWatermarkEstimatorProvider uses to initialize the watermark hold. + All other config is passed via _PollChangeHistoryFn.__init__. + """ + start_time: Timestamp + + +@dataclasses.dataclass +class _QueryRange: + """Lightweight instruction emitted by the polling SDF. + + Contains only the time range to query as Beam Timestamps (int microseconds + internally). Static config (table, project, etc.) is held by + _ExecuteQueryFn which receives these after a Reshuffle commit boundary, + preventing duplicate queries on SDF re-dispatch. + """ + chunk_start: Timestamp + chunk_end: Timestamp + + +class _StreamRestriction: + """Restriction carrying BQ Storage stream names for cross-worker safety. + + Unlike a plain OffsetRange(0, N), this restriction is self-contained: + each split carries the actual stream name strings so it can be processed + on any worker. Composes an OffsetRange for offset logic. + """ + __slots__ = ('stream_names', 'range') + + def __init__( + self, stream_names: Tuple[str, ...], start: int, stop: int) -> None: + self.stream_names = stream_names # tuple of BQ stream name strings + self.range = OffsetRange(start, stop) + + @property + def start(self) -> int: + return self.range.start + + @property + def stop(self) -> int: + return self.range.stop + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _StreamRestriction): + return False + return ( + self.stream_names == other.stream_names and self.range == other.range) + + def __hash__(self) -> int: + return hash((type(self), self.stream_names, self.range)) + + def __repr__(self) -> str: + return ( + '_StreamRestriction(streams=%d, start=%d, stop=%d)' % + (len(self.stream_names), self.start, self.stop)) + + def size(self) -> int: + return self.range.size() + + +class _StreamRestrictionTracker(beam.io.iobase.RestrictionTracker): + """Tracker for _StreamRestriction, delegating offset logic to + OffsetRestrictionTracker.""" + def __init__(self, restriction: _StreamRestriction) -> None: + self._stream_names = restriction.stream_names + self._offset_tracker = OffsetRestrictionTracker(restriction.range) + + def current_restriction(self) -> _StreamRestriction: + r = self._offset_tracker.current_restriction() + return _StreamRestriction(self._stream_names, r.start, r.stop) + + def try_claim(self, position: int) -> bool: + return self._offset_tracker.try_claim(position) + + def try_split( + self, fraction_of_remainder: float + ) -> Optional[Tuple[_StreamRestriction, _StreamRestriction]]: + result = self._offset_tracker.try_split(fraction_of_remainder) + if result is not None: + primary, residual = result + return ( + _StreamRestriction(self._stream_names, primary.start, primary.stop), + _StreamRestriction(self._stream_names, residual.start, residual.stop)) + return None + + def check_done(self) -> None: + self._offset_tracker.check_done() + + def current_progress(self): + return self._offset_tracker.current_progress() + + def is_bounded(self) -> bool: + return True + + +class _NonSplittableOffsetTracker(OffsetRestrictionTracker): + """OffsetRestrictionTracker that allows checkpointing but prevents splitting. + + Checkpointing (fraction=0) is required for defer_remainder(). All other + split fractions are refused, ensuring the polling SDF runs as a singleton. + """ + def try_split( + self, fraction_of_remainder: float + ) -> Optional[Tuple[OffsetRange, OffsetRange]]: + if fraction_of_remainder == 0: + return super().try_split(fraction_of_remainder) + return None + + +class _PollWatermarkEstimator(WatermarkEstimator): + """Watermark estimator that tracks both a watermark hold and poll cursor. + + The watermark hold (reported via current_watermark) is set to start_ts: + the earliest data timestamp emitted by the current poll. This prevents + downstream stages from seeing data as late. + + The poll cursor (last_end) tracks where the next poll should start. + This is separate from the watermark so we can hold the watermark back + at start_ts while still advancing the poll cursor to end_ts. + + All timestamps are Beam Timestamps (int microseconds internally). + + State is checkpointed as (watermark_hold, last_end) so + both values survive SDF re-dispatch. + """ + def __init__(self, state: Tuple[Timestamp, Timestamp]) -> None: + self._watermark_hold, self._last_end = state + + def observe_timestamp(self, timestamp: Timestamp) -> None: + pass + + def current_watermark(self) -> Timestamp: + return self._watermark_hold + + def get_estimator_state(self) -> Tuple[Timestamp, Timestamp]: + return (self._watermark_hold, self._last_end) + + def set_watermark(self, timestamp: Timestamp) -> None: + if not isinstance(timestamp, Timestamp): + raise ValueError('set_watermark expects a Timestamp as input') + if self._watermark_hold and self._watermark_hold > timestamp: + raise ValueError( + 'Watermark must be monotonically increasing. ' + 'Provided %s < current %s' % (timestamp, self._watermark_hold)) + self._watermark_hold = timestamp + + def advance_poll_cursor(self, end: Timestamp) -> None: + """Record end so the next poll starts from here. + + Only advances forward: if end is earlier than the current cursor + (e.g. BQ clock regression), the cursor stays put so the next poll + doesn't re-query an already-covered range. + """ + self._last_end = max(self._last_end, end) + + def poll_cursor(self) -> Timestamp: + """Return the start Timestamp for the next poll.""" + return self._last_end + + +class _PollWatermarkEstimatorProvider(WatermarkEstimatorProvider): + """Provider for _PollWatermarkEstimator. + + Initializes with watermark hold at start_time and poll cursor at + start_time (first poll will query from start_time). + """ + def initial_estimator_state( + self, element: _PollConfig, + restriction: OffsetRange) -> Tuple[Timestamp, Timestamp]: + return (element.start_time, element.start_time) + + def create_watermark_estimator( + self, estimator_state: Tuple[Timestamp, + Timestamp]) -> _PollWatermarkEstimator: + return _PollWatermarkEstimator(estimator_state) + + +def _table_key(table_ref: 'bigquery.TableReference') -> str: + """Convert a TableReference to a 'project.dataset.table' string.""" + return f'{table_ref.projectId}.{table_ref.datasetId}.{table_ref.tableId}' + + +def build_changes_query( + table: str, + start: Timestamp, + end: Timestamp, + change_function: str, + change_type_column: str = 'change_type', + change_timestamp_column: str = 'change_timestamp', + columns: Optional[List[str]] = None, + row_filter: Optional[str] = None) -> str: + """Build a CHANGES() or APPENDS() SQL query. + + Args: + table: Table name as 'project.dataset.table' or 'project:dataset.table'. + start: Start timestamp (Beam Timestamp). Inclusive. + end: End timestamp (Beam Timestamp). Exclusive. + change_function: 'CHANGES' or 'APPENDS'. + change_type_column: Output column name for _CHANGE_TYPE pseudo-column. + change_timestamp_column: Output column name for _CHANGE_TIMESTAMP + pseudo-column. + columns: Optional list of column names to select. If None, selects all + columns. Pseudo-columns are always appended regardless. + row_filter: Optional SQL WHERE clause (without the WHERE keyword). + Applied after the CHANGES/APPENDS function. + + Returns: + SQL string. + """ + # Normalize 'project:dataset.table' to 'project.dataset.table' + table = table.replace(':', '.') + start_iso = start.to_rfc3339() + end_iso = end.to_rfc3339() + # Pseudo-columns (_CHANGE_TYPE, _CHANGE_TIMESTAMP) can't be written to + # destination tables with their original names. Rename them so they can + # be persisted to the temp table for Storage Read API reading. + pseudo = ( + f"_CHANGE_TYPE AS {change_type_column}, " + f"_CHANGE_TIMESTAMP AS {change_timestamp_column}") + if columns is None: + select = f"SELECT * EXCEPT(_CHANGE_TYPE, _CHANGE_TIMESTAMP), {pseudo}" + else: + select = f"SELECT {', '.join(columns)}, {pseudo}" + from_clause = ( + f"FROM {change_function}" + f"(TABLE `{table}`, " + f"TIMESTAMP '{start_iso}', " + f"TIMESTAMP '{end_iso}')") + where = f" WHERE {row_filter}" if row_filter else "" + return f"{select} {from_clause}{where}" + + +def compute_ranges(start: Timestamp, end: Timestamp, + change_function: str) -> List[Tuple[Timestamp, Timestamp]]: + """Split [start, end) into query-safe chunks. + + CHANGES() has a max 1-day range. APPENDS() has no limit. + + Args: + start: Start Timestamp. Inclusive. + end: End Timestamp. Exclusive. + change_function: 'CHANGES' or 'APPENDS'. + + Returns: + List of (start, end) Timestamp tuples. Empty if end <= start. + """ + if end <= start: + return [] + + if change_function != 'CHANGES': + return [(start, end)] + + # CHANGES: chunk into <=1-day ranges + ranges = [] + current = start + while current < end: + chunk_end = min(current + _MAX_CHANGES_RANGE, end) + ranges.append((current, chunk_end)) + current = chunk_end + return ranges + + +def _utc(ts: Timestamp) -> str: + """Format a Beam Timestamp as a concise UTC string for logging.""" + return ts.to_utc_datetime(has_tz=True).strftime('%Y-%m-%dT%H:%M:%S.%f') + + +# ============================================================================= +# Poll: _PollChangeHistoryFn (Polling SDF) +# ============================================================================= + + +class _PollChangeHistoryFn(beam.DoFn, beam.transforms.core.RestrictionProvider): + """SDF that periodically emits _QueryRange instructions. + + Uses defer_remainder() for poll timing and _PollWatermarkEstimator to + control the watermark. The watermark is initially held at start_time, then + advanced to start_ts of each poll. + + All timestamps are Beam Timestamps (int microseconds internally). + Durations (buffer, poll_interval) are Beam Durations. + + Derives start_ts from the poll cursor. On each poll: + 1. start_ts = poll cursor (last end_ts, or start_time on first poll) + 2. end_ts = bq_now - buffer + 3. Computes query chunks, yields _QueryRange per chunk + 4. Advances poll cursor to end_ts (for next poll's start) + 5. Advances watermark to start_ts (earliest data in this poll) + 6. Defers to next poll interval + """ + def __init__( + self, + table: str, + project: str, + change_function: str, + buffer: Duration, + start_time: Timestamp, + stop_time: Timestamp, + poll_interval: Duration, + location: Optional[str] = None) -> None: + self._table = table + self._project = project + self._change_function = change_function + self._buffer = buffer + self._start_time = start_time + self._stop_time = stop_time + self._poll_interval = poll_interval + self._location = location + + def setup(self) -> None: + self._bq_wrapper = bigquery_tools.BigQueryWrapper() + if self._location is None: + table_ref = bigquery_tools.parse_table_reference( + self._table, project=self._project) + self._location = self._bq_wrapper.get_table_location( + table_ref.projectId, table_ref.datasetId, table_ref.tableId) + _LOGGER.info( + '[Poll] Inferred location=%s from source table %s', + self._location, + self._table) + + @retry.with_exponential_backoff( + num_retries=3, + retry_filter=retry.retry_on_server_errors_and_timeout_filter) + def _get_bq_timestamp(self) -> Timestamp: + """Query BigQuery for the current server timestamp. + + Returns a Beam Timestamp created from integer microseconds. + Uses BQ's CURRENT_TIMESTAMP instead of the local clock to avoid + data loss from clock skew between the worker VM and BigQuery. + """ + request = bigquery.BigqueryJobsQueryRequest( + projectId=self._project, + queryRequest=bigquery.QueryRequest( + query='SELECT UNIX_MICROS(CURRENT_TIMESTAMP()) AS ts', + useLegacySql=False, + location=self._location)) + response = self._bq_wrapper.client.jobs.Query(request) + return Timestamp(micros=int(response.rows[0].f[0].v.string_value)) + + def initial_restriction(self, element: _PollConfig) -> OffsetRange: + return OffsetRange(0, sys.maxsize) + + def create_tracker( + self, restriction: OffsetRange) -> _NonSplittableOffsetTracker: + # Guarantee at least one poll cycle: restriction.start == 0 on the first + # invocation (from initial_restriction). After the first try_claim(0) + + # defer_remainder, subsequent invocations arrive with start >= 1. + if restriction.start > 0 and time.time() >= float(self._stop_time): + _LOGGER.info( + '[Poll] create_tracker: stop_time reached, ' + 'returning empty range to terminate SDF') + return _NonSplittableOffsetTracker( + OffsetRange(restriction.start, restriction.start)) + return _NonSplittableOffsetTracker(restriction) + + def restriction_size( + self, element: _PollConfig, restriction: OffsetRange) -> int: + return 1 + + def split(self, element: _PollConfig, + restriction: OffsetRange) -> Iterable[OffsetRange]: + yield restriction + + def truncate(self, element: _PollConfig, restriction: OffsetRange) -> None: + return None + + def _next_poll_time(self, start_ts: Timestamp, + now: float) -> Optional[Timestamp]: + """Return a Timestamp to defer to, or None if we should poll now.""" + earliest = start_ts + self._buffer + self._poll_interval + if now < float(earliest): + return earliest + return None + + def _emit_query_ranges( + self, + start_ts: Timestamp, + end_ts: Timestamp, + watermark_estimator: _PollWatermarkEstimator) -> Iterable[_QueryRange]: + """Compute and yield _QueryRange elements, advancing estimator state.""" + ranges = compute_ranges(start_ts, end_ts, self._change_function) + _LOGGER.info( + '[Poll] %d chunks for [%s, %s)', + len(ranges), + _utc(start_ts), + _utc(end_ts)) + Metrics.counter('BigQueryChangeHistory', 'polls').inc() + + watermark_estimator.advance_poll_cursor(end_ts) + watermark_estimator.set_watermark(start_ts) + _LOGGER.info( + '[Poll] Watermark=%s (start_ts), cursor=%s (end_ts)', + _utc(start_ts), + _utc(end_ts)) + + for chunk_start, chunk_end in ranges: + yield TimestampedValue( + _QueryRange(chunk_start=chunk_start, chunk_end=chunk_end), start_ts) + + @beam.DoFn.unbounded_per_element() + def process( + self, + _: _PollConfig, + restriction_tracker=beam.DoFn.RestrictionParam(), + watermark_estimator=beam.DoFn.WatermarkEstimatorParam( + _PollWatermarkEstimatorProvider()) + ) -> Iterable[_QueryRange]: + + now = time.time() + start_ts = watermark_estimator.poll_cursor() + + defer_to = self._next_poll_time(start_ts, now) + if defer_to is not None: + restriction_tracker.defer_remainder(defer_to) + return + + # Use BQ server time instead of local clock to avoid data loss + # from clock skew between the worker VM and BigQuery. + bq_now = self._get_bq_timestamp() + end_ts = min(bq_now - self._buffer, self._stop_time) + + _LOGGER.info( + '[Poll] Polling: start=%s, end=%s, watermark=%s, ' + 'clock_skew=%.3fs', + _utc(start_ts), + _utc(end_ts), + _utc(watermark_estimator.current_watermark()), + float(bq_now) - now) + + current_index = restriction_tracker.current_restriction().start + + if not restriction_tracker.try_claim(current_index): + return + restriction_tracker.defer_remainder(Timestamp.of(now) + self._poll_interval) + + yield from self._emit_query_ranges(start_ts, end_ts, watermark_estimator) + + +class _ExecuteQueryFn(beam.DoFn): + """Executes a BQ CHANGES/APPENDS query from a _QueryRange instruction. + """ + def __init__( + self, + table: str, + project: str, + change_function: str, + temp_dataset: str, + location: Optional[str], + change_type_column: str = 'change_type', + change_timestamp_column: str = 'change_timestamp', + columns: Optional[List[str]] = None, + row_filter: Optional[str] = None) -> None: + self._table = table + self._project = project + self._change_function = change_function + self._temp_dataset = temp_dataset + self._location = location + self._change_type_column = change_type_column + self._change_timestamp_column = change_timestamp_column + self._columns = columns + self._row_filter = row_filter + + def setup(self) -> None: + self._bq_wrapper = bigquery_tools.BigQueryWrapper() + if self._location is None: + table_ref = bigquery_tools.parse_table_reference( + self._table, project=self._project) + self._location = self._bq_wrapper.get_table_location( + table_ref.projectId, table_ref.datasetId, table_ref.tableId) + _LOGGER.info( + '[Query] Inferred location=%s from source table %s', + self._location, + self._table) + self._get_or_create_temp_dataset() + + def _get_or_create_temp_dataset(self) -> None: + """Create the temp dataset if it doesn't exist. + + Sets a default table expiration so orphaned temp tables (e.g. from + pipeline crashes before cleanup) are automatically garbage-collected. + """ + try: + self._bq_wrapper.client.datasets.Get( + bigquery.BigqueryDatasetsGetRequest( + projectId=self._project, datasetId=self._temp_dataset)) + except HttpError as e: + if e.status_code != 404: + raise + _LOGGER.info( + '[Query] Creating temp dataset %s:%s (location=%s)', + self._project, self._temp_dataset, self._location) + dataset_ref = bigquery.DatasetReference( + projectId=self._project, datasetId=self._temp_dataset) + dataset = bigquery.Dataset( + datasetReference=dataset_ref, + defaultTableExpirationMs=_DEFAULT_TABLE_EXPIRATION_MS) + if self._location is not None: + dataset.location = self._location + self._bq_wrapper.client.datasets.Insert( + bigquery.BigqueryDatasetsInsertRequest( + projectId=self._project, dataset=dataset)) + + def process(self, qr: _QueryRange) -> Iterable[_QueryResult]: + """Execute the BQ query described by a _QueryRange and yield _QueryResult. + """ + + sql = build_changes_query( + self._table, + qr.chunk_start, + qr.chunk_end, + self._change_function, + self._change_type_column, + self._change_timestamp_column, + self._columns, + self._row_filter) + temp_table_id = f'beam_ch_temp_{uuid.uuid4().hex[:8]}' + job_id = f'beam_ch_{uuid.uuid4().hex[:12]}' + + _LOGGER.info( + '[Query] job_id=%s, temp_table=%s.%s, range=[%s, %s)', + job_id, + self._temp_dataset, + temp_table_id, + _utc(qr.chunk_start), + _utc(qr.chunk_end)) + + temp_table_ref = bigquery.TableReference( + projectId=self._project, + datasetId=self._temp_dataset, + tableId=temp_table_id) + + reference = bigquery.JobReference( + jobId=job_id, projectId=self._project, location=self._location) + + request = bigquery.BigqueryJobsInsertRequest( + projectId=self._project, + job=bigquery.Job( + configuration=bigquery.JobConfiguration( + query=bigquery.JobConfigurationQuery( + query=sql, + useLegacySql=False, + destinationTable=temp_table_ref, + writeDisposition='WRITE_TRUNCATE', + ), + ), + jobReference=reference)) + + _LOGGER.info('[Query] Submitting BQ job %s...', job_id) + response = self._bq_wrapper._start_job(request) + _LOGGER.info('[Query] BQ job %s submitted, waiting...', job_id) + self._bq_wrapper.wait_for_bq_job( + response.jobReference, sleep_duration_sec=2) + _LOGGER.info( + '[Query] BQ job %s DONE. Results in %s.%s', + job_id, + self._temp_dataset, + temp_table_id) + Metrics.counter('BigQueryChangeHistory', 'queries').inc() + + yield _QueryResult( + temp_table_ref=temp_table_ref, + range_start=qr.chunk_start, + range_end=qr.chunk_end) + + +class _CDCWatermarkEstimatorProvider(WatermarkEstimatorProvider): + """WatermarkEstimatorProvider that initializes the hold from _QueryResult. + + Uses range_start from the element to set the initial watermark hold. + This prevents the runner from advancing the watermark past the data's + timestamps before any rows are emitted. + """ + def initial_estimator_state( + self, element: _QueryResult, + restriction: _StreamRestriction) -> Timestamp: + return element.range_start + + def create_watermark_estimator( + self, estimator_state: Timestamp) -> ManualWatermarkEstimator: + return ManualWatermarkEstimator(estimator_state) + + +# ============================================================================= +# Read: _ReadStorageStreamsSDF +# ============================================================================= + + +class _ReadStorageStreamsSDF(beam.DoFn, + beam.transforms.core.RestrictionProvider): + """SDF that reads a temp table via BigQuery Storage Read API. + + Note on SDF lifecycle: the runner decomposes this SDF into three internal + wrapper DoFns, each a separately deserialized copy: + - Stage A (PairWithRestriction): calls initial_restriction(): no setup() + - Stage B (SplitAndSizeRestrictions): calls split(), restriction_size() + - Stage C (ProcessSizedElements): calls setup(), then process() + Because initial_restriction() runs on a different copy than process(), + _ensure_client() lazily creates a gRPC client on whichever copy needs one. + The _StreamRestriction carries stream names directly so no shared state + is needed between copies. + + Each element is a _QueryResult pointing to a temp table. + + Watermark: Uses ManualWatermarkEstimator so the watermark only advances + as fast as the change-timestamp values we emit. + + Emits: + Main output: TimestampedValue(row_dict, event_timestamp) + Side output (_CLEANUP_TAG): (table_key, (streams_read, total_streams)) + """ + def __init__( + self, + batch_arrow_read: bool = True, + change_timestamp_column: str = 'change_timestamp', + max_split_rounds: int = 1, + emit_raw_batches: bool = False) -> None: + self._batch_arrow_read = batch_arrow_read + self._change_timestamp_column = change_timestamp_column + self._max_split_rounds = max_split_rounds + self._emit_raw_batches = emit_raw_batches + self._storage_client = None + + def _ensure_client(self) -> None: + """Lazily initialize the Storage client. + + Called from both setup() and initial_restriction() because the runner + may invoke initial_restriction on the RestrictionProvider instance + before setup() runs (or on a separately deserialized copy). + """ + if self._storage_client is None: + _LOGGER.info('[Read] creating BigQueryReadClient') + self._storage_client = bq_storage.BigQueryReadClient() + + def setup(self) -> None: + self._ensure_client() + + def _split_all_streams(self, stream_names: Tuple[str, ...], + max_split_rounds: int) -> Tuple[str, ...]: + """Split each stream at fraction=0.5 for up to max_split_rounds rounds. + + Each round attempts to split every stream in the current list. A + successful split replaces the original stream with primary + remainder. + A refused split (both fields empty) keeps the original stream intact. + Stops when max_split_rounds is reached or a full round produces zero + new splits. + + BQ's server-side granularity controls how many splits are possible. + Small tables may not split at all; large tables may allow multiple + rounds of doubling. + """ + result = list(stream_names) + for round_num in range(1, max_split_rounds + 1): + new_result = [] + made_progress = False + for name in result: + response = self._storage_client.split_read_stream( + request=bq_storage.types.SplitReadStreamRequest( + name=name, fraction=0.5)) + primary = response.primary_stream.name + remainder = response.remainder_stream.name + if primary and remainder: + new_result.extend([primary, remainder]) + made_progress = True + else: + new_result.append(name) + result = new_result + _LOGGER.info( + '[Read] _split_all_streams round %d/%d: %d streams ' + '(progress=%s)', + round_num, + max_split_rounds, + len(result), + made_progress) + if not made_progress: + break + return tuple(result) + + def initial_restriction(self, element: _QueryResult) -> _StreamRestriction: + """Create ReadSession and return _StreamRestriction with stream names. + + When max_split_rounds > 0, uses SplitReadStream to subdivide each + stream at fraction=0.5 for up to max_split_rounds rounds, maximizing + parallelism beyond what CreateReadSession provides. + """ + self._ensure_client() + table_key = _table_key(element.temp_table_ref) + session = self._create_read_session(element.temp_table_ref) + stream_names = tuple(s.name for s in session.streams) + original_count = len(stream_names) + _LOGGER.info( + '[Read] initial_restriction for %s: %d streams from CreateReadSession', + table_key, + original_count) + + if self._max_split_rounds > 0: + stream_names = self._split_all_streams( + stream_names, self._max_split_rounds) + _LOGGER.info( + '[Read] initial_restriction for %s: %d -> %d streams ' + 'after SplitReadStream', + table_key, + original_count, + len(stream_names)) + + return _StreamRestriction(stream_names, 0, len(stream_names)) + + def create_tracker( + self, restriction: _StreamRestriction) -> _StreamRestrictionTracker: + return _StreamRestrictionTracker(restriction) + + def restriction_size( + self, element: _QueryResult, restriction: _StreamRestriction) -> int: + return restriction.size() + + def split(self, element: _QueryResult, + restriction: _StreamRestriction) -> Iterable[_StreamRestriction]: + """Yield one _StreamRestriction per stream for parallel distribution.""" + if restriction.size() <= 1: + yield restriction + else: + for i in range(restriction.start, restriction.stop): + yield _StreamRestriction(restriction.stream_names, i, i + 1) + + def is_bounded(self) -> bool: + return True + + def process( + self, + element: _QueryResult, + restriction_tracker=beam.DoFn.RestrictionParam(), + watermark_estimator=beam.DoFn.WatermarkEstimatorParam( + _CDCWatermarkEstimatorProvider()) + ): + self._ensure_client() + table_key = _table_key(element.temp_table_ref) + + _LOGGER.info( + '[Read] Processing %s, range=[%s, %s), ' + 'initial watermark=%s', + table_key, + _utc(element.range_start), + _utc(element.range_end), + _utc(watermark_estimator.current_watermark())) + + restriction = restriction_tracker.current_restriction() + stream_names = restriction.stream_names + total_streams = len(stream_names) + + streams_read = 0 + + _LOGGER.info( + '[Read] Reading streams [%d, %d) of %d total for %s', + restriction.start, + restriction.stop, + total_streams, + table_key) + + for i in range(restriction.start, restriction.stop): + if not restriction_tracker.try_claim(i): + _LOGGER.info( + '[Read] try_claim(%d) FAILED for %s: ' + 'runner split or checkpoint, breaking', + i, + table_key) + break + + stream_name = stream_names[i] + _LOGGER.info( + '[Read] try_claim(%d) succeeded: reading stream %s', i, stream_name) + + if self._emit_raw_batches: + stream_batches = 0 + for raw_batch in self._read_stream_raw(stream_name): + yield TimestampedValue(raw_batch, element.range_start) + stream_batches += 1 + Metrics.counter( + 'BigQueryChangeHistory', 'batches_emitted').inc(stream_batches) + _LOGGER.info( + '[Read] Finished reading stream %d for %s: %d batches', + i, table_key, stream_batches) + else: + stream_rows = 0 + for row in self._read_stream(stream_name): + ts = row.get(self._change_timestamp_column) + if ts is None: + raise ValueError( + 'Row missing %r column. Row keys: %s' % + (self._change_timestamp_column, list(row.keys()))) + if isinstance(ts, datetime.datetime): + ts = Timestamp.from_utc_datetime(ts) + + yield TimestampedValue(row, ts) + stream_rows += 1 + Metrics.counter( + 'BigQueryChangeHistory', 'rows_emitted').inc(stream_rows) + _LOGGER.info( + '[Read] Finished reading stream %d for %s: %d rows', + i, table_key, stream_rows) + + streams_read += 1 + Metrics.counter('BigQueryChangeHistory', 'streams_read').inc() + + # Advance watermark to range_end after reading all streams. The + # initial hold was set to range_start by _CDCWatermarkEstimatorProvider. + watermark_estimator.set_watermark(element.range_end) + _LOGGER.info( + '[Read] Watermark advanced to %s (range_end) for %s', + _utc(element.range_end), + table_key) + + # Release the storage client so the gRPC channel doesn't go stale + # between process() calls. _ensure_client() will create a fresh one. + self._storage_client = None + + # Emit cleanup signal. Every split that reads at least one stream + # reports how many it read. + if streams_read > 0: + _LOGGER.info( + '[Read] Emitting cleanup signal for %s: ' + 'streams_read=%d, total_streams=%d', + table_key, + streams_read, + total_streams) + yield beam.pvalue.TaggedOutput( + _CLEANUP_TAG, (table_key, (streams_read, total_streams))) + + def _create_read_session(self, table_ref: 'bigquery.TableReference') -> Any: + """Create a BigQuery Storage ReadSession for the given table.""" + table_path = ( + f'projects/{table_ref.projectId}/' + f'datasets/{table_ref.datasetId}/' + f'tables/{table_ref.tableId}') + + requested_session = bq_storage.types.ReadSession() + requested_session.table = table_path + requested_session.data_format = bq_storage.types.DataFormat.ARROW + read_options = requested_session.read_options + read_options.arrow_serialization_options.buffer_compression = ( + bq_storage.types.ArrowSerializationOptions.CompressionCodec.ZSTD) + + session = self._storage_client.create_read_session( + parent=f'projects/{table_ref.projectId}', + read_session=requested_session, + max_stream_count=_DEFAULT_MAX_STREAMS) + _LOGGER.info( + '[Read] _create_read_session: table=%s, %d streams', + table_path, + len(session.streams)) + return session + + def _read_stream(self, stream_name: str) -> Iterable[Dict[str, Any]]: + """Read all rows from a single Storage API stream as dicts. + + When batch_arrow_read is enabled, converts entire Arrow RecordBatches + at once using to_pylist() instead of calling .as_py() on each cell + individually. This is ~1.5x faster for large tables at the cost of ~2x + peak memory per batch. + """ + if self._batch_arrow_read: + yield from self._read_stream_batch(stream_name) + else: + yield from self._read_stream_row_by_row(stream_name) + + def _read_stream_row_by_row(self, + stream_name: str) -> Iterable[Dict[str, Any]]: + """Row-by-row Arrow conversion (lower memory than batch mode).""" + t0 = time.time() + row_count = 0 + for row in self._storage_client.read_rows(stream_name).rows(): + yield dict((item[0], item[1].as_py()) for item in row.items()) + row_count += 1 + elapsed = time.time() - t0 + _LOGGER.info( + '[Read] row_by_row: %d rows in %.2fs (%.0f rows/s)', + row_count, + elapsed, + row_count / elapsed if elapsed > 0 else 0) + + def _read_stream_batch(self, stream_name: str) -> Iterable[Dict[str, Any]]: + """Batch-convert Arrow RecordBatches for high throughput.""" + schema = None + row_count = 0 + t0 = time.time() + for response in self._storage_client.read_rows(stream_name): + if schema is None and response.arrow_schema.serialized_schema: + schema = pyarrow.ipc.read_schema( + pyarrow.py_buffer(response.arrow_schema.serialized_schema)) + batch_bytes = response.arrow_record_batch.serialized_record_batch + if batch_bytes and schema is not None: + batch = pyarrow.ipc.read_record_batch( + pyarrow.py_buffer(batch_bytes), schema) + yield from batch.to_pylist() + row_count += batch.num_rows + elapsed = time.time() - t0 + _LOGGER.info( + '[Read] batch_read: %d rows in %.2fs (%.0f rows/s)', + row_count, + elapsed, + row_count / elapsed if elapsed > 0 else 0) + + def _read_stream_raw( + self, + stream_name: str) -> Iterable[Tuple[bytes, bytes]]: + """Yield raw (schema_bytes, batch_bytes) without decompression. + + Used when emit_raw_batches is enabled to defer decompression and + Arrow-to-Python conversion to a downstream DoFn after reshuffling. + Schema bytes are included in each tuple so each batch is + self-contained and can be decoded independently. + """ + schema_bytes = b'' + batch_count = 0 + t0 = time.time() + for response in self._storage_client.read_rows(stream_name): + if not schema_bytes and response.arrow_schema.serialized_schema: + schema_bytes = bytes(response.arrow_schema.serialized_schema) + batch_bytes = response.arrow_record_batch.serialized_record_batch + if batch_bytes and schema_bytes: + yield (schema_bytes, bytes(batch_bytes)) + batch_count += 1 + elapsed = time.time() - t0 + _LOGGER.info( + '[Read] raw_read: %d batches in %.2fs', + batch_count, + elapsed) + + +class _DecompressArrowBatchesFn(beam.DoFn): + """Decompress and convert raw Arrow batches to timestamped row dicts. + + Receives GBK output: (shard_key, Iterable[(schema_bytes, batch_bytes)]) + and converts each batch to individual row dicts with event timestamps + extracted from the change_timestamp column. + """ + def __init__(self, change_timestamp_column: str = 'change_timestamp') -> None: + self._change_timestamp_column = change_timestamp_column + + def process( + self, + element: Tuple[int, Iterable[Tuple[bytes, bytes]]] + ) -> Iterable[Dict[str, Any]]: + _, batches = element + for schema_bytes, batch_bytes in batches: + schema = pyarrow.ipc.read_schema(pyarrow.py_buffer(schema_bytes)) + batch = pyarrow.ipc.read_record_batch( + pyarrow.py_buffer(batch_bytes), schema) + + rows = batch.to_pylist() + for row in rows: + ts = row.get(self._change_timestamp_column) + if ts is None: + raise ValueError( + 'Row missing %r column. Row keys: %s' % + (self._change_timestamp_column, list(row.keys()))) + if isinstance(ts, datetime.datetime): + ts = Timestamp.from_utc_datetime(ts) + yield TimestampedValue(row, ts) + Metrics.counter('BigQueryChangeHistory', 'rows_emitted').inc(len(rows)) + + +# ============================================================================= +# Cleanup: _CleanupTempTablesFn +# ============================================================================= + + +class _CleanupTempTablesFn(beam.DoFn): + """Stateful DoFn that deletes temp tables after all streams are read. + + Receives cleanup signals from the Read SDF as: + (table_key, (streams_read_count, total_streams)) + + Accumulates streams_read across all signals for the same table_key. + When streams_read >= total_streams, deletes the temp table. The >= + (rather than ==) guards against duplicate delivery in at-least-once runners. + """ + STREAMS_READ = beam.transforms.userstate.CombiningValueStateSpec( + 'streams_read', sum) + + def setup(self) -> None: + _LOGGER.info('[Cleanup] setup: creating BigQueryWrapper') + self._bq_wrapper = bigquery_tools.BigQueryWrapper() + + def process( + self, + element: Tuple[str, Tuple[int, int]], + streams_read=beam.DoFn.StateParam(STREAMS_READ) + ) -> None: + table_key = element[0] + split_count = element[1][0] + total_streams = element[1][1] + + _LOGGER.info( + '[Cleanup] Received cleanup signal for %s: ' + 'split_count=%d, total_streams=%d', + table_key, + split_count, + total_streams) + + streams_read.add(split_count) + current_read = streams_read.read() + + _LOGGER.info( + '[Cleanup] State for %s: streams_read=%d/%d', + table_key, + current_read, + total_streams) + + if current_read >= total_streams: + parts = table_key.split('.') + if len(parts) == 3: + project, dataset, table = parts + _LOGGER.info( + '[Cleanup] All streams read: DELETING temp table %s', table_key) + self._bq_wrapper._delete_table(project, dataset, table) + _LOGGER.info('[Cleanup] Deleted temp table %s', table_key) + Metrics.counter('BigQueryChangeHistory', 'temp_tables_deleted').inc() + streams_read.clear() + else: + _LOGGER.info( + '[Cleanup] Not yet complete for %s (%d/%d), ' + 'waiting for more signals', + table_key, + current_read, + total_streams) + + +# ============================================================================= +# Public API: ReadBigQueryChangeHistory +# ============================================================================= + + +class ReadBigQueryChangeHistory(beam.PTransform): + """Streaming source for BigQuery change history. + + Continuously polls BigQuery APPENDS() or CHANGES() functions and emits + changed rows as an unbounded PCollection of dicts. + + Args: + table: BigQuery table to read changes from. + Format: 'project:dataset.table' or 'project.dataset.table'. + poll_interval_sec: Seconds between polls. Default 60. + start_time: Start reading from this timestamp (float, epoch seconds). + Default: current time when pipeline starts. + stop_time: Stop polling at this timestamp. Default: run forever. + change_function: 'CHANGES' or 'APPENDS'. Default 'APPENDS'. + buffer_sec: Safety buffer in seconds behind now(). Default 10. BQ does not + fail or wait if the query end_ts is less than BQ's CURRENT_TIMESTAMP. + This is an extra guardrail to protect against silent data. + project: GCP project ID. Default: from pipeline options. + temp_dataset: Dataset for temp tables. If None (default), a + per-pipeline dataset is auto-created with a 24-hour table + expiration as a safety net for orphaned tables. Set this to + use an existing dataset (e.g. if your service account lacks + bigquery.datasets.create permission). + location: BigQuery geographic location for query jobs and temp + dataset (e.g. 'US', 'us-central1'). If None (default), inferred + from the source table. + change_type_column: Output column name for the _CHANGE_TYPE + pseudo-column. Default 'change_type'. Change this if your source + table already has a column named 'change_type'. + change_timestamp_column: Output column name for the + _CHANGE_TIMESTAMP pseudo-column. Default 'change_timestamp'. + Change this if your source table already has a column named + 'change_timestamp'. This column is also used internally to + extract event timestamps for watermark tracking. + columns: Optional list of column names to select from the source + table. If None (default), all columns are selected. The + pseudo-columns (change_type, change_timestamp) are always + included regardless of this setting. + row_filter: Optional SQL boolean expression used as a WHERE clause + on the CHANGES/APPENDS query. Do not include the WHERE keyword. + Example: ``'status = "active" AND region = "US"'``. + batch_arrow_read: If True (default), convert Arrow RecordBatches in + bulk using to_pylist() instead of per-cell .as_py() calls. + This is 1.5x faster for large tables at the cost of ~2x peak + memory per RecordBatch. Set to False for minimal memory usage. + max_split_rounds: Maximum number of recursive SplitReadStream + rounds. Each round splits every stream at fraction=0.5, + potentially doubling the stream count (if BQ allows). Default + 1 (one round of splitting). Set 0 to disable splitting + entirely. Set higher for very large tables where more + parallelism is needed. + decompress_shards: If set to a positive integer, the Read SDF + emits raw compressed Arrow batches instead of decoded rows. + The batches are reshuffled for fan-out and then decoded in a + separate DoFn. This spreads decompression and Arrow-to-Python + conversion CPU across more workers. If None (default), rows + are decoded inline within the Read SDF. + """ + def __init__( + self, + table: str, + poll_interval_sec: float = 60, + start_time: Optional[float] = None, + stop_time: Optional[float] = None, + change_function: str = 'APPENDS', + buffer_sec: float = 10, + project: Optional[str] = None, + temp_dataset: Optional[str] = None, + location: Optional[str] = None, + change_type_column: str = 'change_type', + change_timestamp_column: str = 'change_timestamp', + columns: Optional[List[str]] = None, + row_filter: Optional[str] = None, + batch_arrow_read: bool = True, + max_split_rounds: int = 1, + decompress_shards: Optional[int] = None) -> None: + super().__init__() + if bq_storage is None: + raise ImportError( + 'google-cloud-bigquery-storage is required for ' + 'ReadBigQueryChangeHistory. Install it with: ' + 'pip install google-cloud-bigquery-storage') + if pyarrow is None: + raise ImportError( + 'pyarrow is required for ReadBigQueryChangeHistory. ' + 'Install it with: pip install pyarrow') + if change_function not in ('CHANGES', 'APPENDS'): + raise ValueError( + f"change_function must be 'CHANGES' or 'APPENDS', " + f"got '{change_function}'") + if poll_interval_sec < 15: + raise ValueError( + f'poll_interval_sec must be >= 15, got {poll_interval_sec}') + if buffer_sec < 0: + raise ValueError(f'buffer_sec must be >= 0, got {buffer_sec}') + self._table = table + self._poll_interval_sec = poll_interval_sec + self._start_time = start_time + self._stop_time = stop_time + self._change_function = change_function + self._buffer_sec = buffer_sec + self._project = project + self._temp_dataset = temp_dataset + self._location = location + self._change_type_column = change_type_column + self._change_timestamp_column = change_timestamp_column + self._columns = columns + self._row_filter = row_filter + self._batch_arrow_read = batch_arrow_read + self._max_split_rounds = max_split_rounds + self._decompress_shards = decompress_shards + + def expand(self, pbegin: beam.pvalue.PBegin) -> beam.PCollection: + project = self._project + if project is None: + project = pbegin.pipeline.options.view_as( + beam.options.pipeline_options.GoogleCloudOptions).project + + if project is None: + raise ValueError( + 'project must be specified either in ReadBigQueryChangeHistory ' + 'or in pipeline options (--project)') + + start_time = Timestamp(self._start_time or time.time()) + stop_time = ( + Timestamp(self._stop_time) + if self._stop_time is not None else MAX_TIMESTAMP) + buffer = Duration(seconds=self._buffer_sec) + poll_interval = Duration(seconds=self._poll_interval_sec) + + temp_dataset = self._temp_dataset + if temp_dataset is None: + temp_dataset = f'beam_ch_temp_{uuid.uuid4().hex[:12]}' + + _LOGGER.info( + '[ReadBigQueryChangeHistory] expand: table=%s, project=%s, ' + 'change_function=%s, poll_interval=%d sec, buffer=%d sec, ' + 'temp_dataset=%s, start_time=%s, stop_time=%s', + self._table, + project, + self._change_function, + self._poll_interval_sec, + self._buffer_sec, + temp_dataset, + _utc(start_time), + _utc(stop_time) if stop_time != MAX_TIMESTAMP else 'INF') + + # Custom polling SDF emits lightweight _QueryRange instructions. + # The SDF uses defer_remainder() for poll timing and + # _PollWatermarkEstimator to hold the watermark at data timestamps. + # On the first invocation it handles the full historical range + # [start_time, now - buffer) in a single poll. + config = _PollConfig(start_time=start_time) + + query_ranges = ( + pbegin + | 'CreatePollConfig' >> beam.Create([config]) + | 'PollChangeHistory' >> beam.ParDo( + _PollChangeHistoryFn( + table=self._table, + project=project, + change_function=self._change_function, + buffer=buffer, + start_time=start_time, + stop_time=stop_time, + poll_interval=poll_interval, + location=self._location))) + + # CommitQueryResults: Reshuffle commits _QueryResult (temp table ref) + # so that if the Read SDF retries, it re-reads the existing temp table + # instead of re-running the BQ query. + # Possible edge-case is that if ReadStorageStreams doesn't read the temp + # table within 24 hours (table expiration) it can end up in a bad state by + # trying to query a non-existing table. + query_results = ( + query_ranges + | 'CommitQueryRanges' >> beam.Reshuffle() + | 'ExecuteQueries' >> beam.ParDo( + _ExecuteQueryFn( + table=self._table, + project=project, + change_function=self._change_function, + temp_dataset=temp_dataset, + location=self._location, + change_type_column=self._change_type_column, + change_timestamp_column=self._change_timestamp_column, + columns=self._columns, + row_filter=self._row_filter)) + | 'CommitQueryResults' >> beam.Reshuffle()) + + emit_raw = self._decompress_shards is not None + + read_sdf = beam.ParDo( + _ReadStorageStreamsSDF( + batch_arrow_read=self._batch_arrow_read, + change_timestamp_column=self._change_timestamp_column, + max_split_rounds=self._max_split_rounds, + emit_raw_batches=emit_raw)) + if emit_raw: + read_sdf = read_sdf.with_output_types(Tuple[bytes, bytes]) + else: + read_sdf = read_sdf.with_output_types(Dict[str, Any]) + + read_outputs = ( + query_results + | 'ReadStorageStreams' >> read_sdf.with_outputs( + _CLEANUP_TAG, main='rows')) + + _ = ( + read_outputs[_CLEANUP_TAG] + | 'CleanupTempTables' >> beam.ParDo(_CleanupTempTablesFn())) + + if emit_raw: + # Fan out raw Arrow batches across decompress_shards workers + # via GBK, then decompress and convert to timestamped row dicts. + # Uses a discarding trigger so GBK fires per-element without + # waiting for the GlobalWindow to close. + num_shards = self._decompress_shards + rows = ( + read_outputs['rows'] + | 'ShardBatches' >> beam.WithKeys( + lambda _, n=num_shards: random.randint(0, n - 1)) + | 'WindowForGBK' >> beam.WindowInto( + GlobalWindows(), + trigger=beam_trigger.Repeatedly( + beam_trigger.AfterCount(1)), + accumulation_mode=( + beam_trigger.AccumulationMode.DISCARDING)) + | 'GroupByShardKey' >> beam.GroupByKey() + | 'DecompressBatches' >> beam.ParDo( + _DecompressArrowBatchesFn( + change_timestamp_column=( + self._change_timestamp_column)))) + return rows + else: + return read_outputs['rows'] diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py new file mode 100644 index 0000000000..a5da3888ee --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/metric.py @@ -0,0 +1,764 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Configurable metric computation for anomaly detection pipelines. + +This module provides a ``MetricSpec`` configuration system and a +``ComputeMetric`` PTransform that computes windowed, grouped metrics from +raw row dicts (e.g., from ``ReadBigQueryChangeHistory``). The output is +suitable for feeding directly into ``AnomalyDetection``. + +Example usage:: + + from bqmonitor.metric import ( + MetricSpec, AggregationSpec, WindowSpec, MeasureSpec, + DerivedField, WindowType, AggOp, ComputeMetric) + from bqmonitor.safe_eval import Expr + from apache_beam.ml.anomaly.transforms import AnomalyDetection + from apache_beam.ml.anomaly.detectors.zscore import ZScore + + # CUJ 1: Total revenue per hour + spec = MetricSpec( + name='revenue', + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=3600), + measures=[MeasureSpec( + field='transaction_amount', agg=AggOp.SUM, alias='revenue')], + ), + ) + result = cdc_rows | ComputeMetric(spec) | AnomalyDetection(ZScore()) + + # CUJ 2: CTR grouped by dimensions + spec = MetricSpec( + name='ctr', + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=86400), + group_by=['campaign_type', 'user_segment'], + measures=[ + MeasureSpec(field='is_click', agg=AggOp.SUM, alias='clicks'), + MeasureSpec(field='is_click', agg=AggOp.COUNT, + alias='impressions'), + ], + ), + measure_combiner=Expr.from_string("clicks / impressions"), + ) + + # CUJ 3: Success rate with derived field + spec = MetricSpec( + name='success_rate', + derived_fields=[ + DerivedField( + name='is_success', + expression=Expr.from_string( + "1 if status == 'success' else 0")), + ], + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=86400), + group_by=['brand_name', 'category'], + measures=[ + MeasureSpec(field='is_success', agg=AggOp.SUM, + alias='successes'), + MeasureSpec(field='is_success', agg=AggOp.COUNT, alias='total'), + ], + ), + measure_combiner=Expr.from_string("successes / total"), + ) +""" + +import dataclasses +import random +from enum import Enum +from typing import Any +from typing import Optional +from typing import Tuple + +import apache_beam as beam +from apache_beam.transforms import combiners +from apache_beam.transforms import window as beam_window + +from bqmonitor.safe_eval import Expr +from apache_beam.ml.anomaly.specifiable import specifiable + + +class WindowType(Enum): + """Window type for metric aggregation.""" + FIXED = 'fixed' + SLIDING = 'sliding' + + +class AggOp(Enum): + """Aggregation operator.""" + SUM = 'SUM' + COUNT = 'COUNT' + MIN = 'MIN' + MAX = 'MAX' + MEAN = 'MEAN' + + +class FanoutStrategy(Enum): + """Strategy for global (non-keyed) aggregation parallelism. + + NONE: Plain CombineGlobally, no fanout. Relies on combiner lifting (PGBK) + for mapper-side pre-combining. Works well when upstream provides enough + parallel bundles (e.g. decompress_shards) and streaming state I/O on a + single key is not a bottleneck. + HOTKEY_FANOUT: Uses CombineGlobally.with_fanout(). Per-bundle nonce sharding. + Better PGBK table efficiency (1 slot per bundle), but shard distribution + depends on bundle count and sizes. Good for multi-key CombinePerKey where + only a few keys are hot. + SHARDED: Per-element random sharding with _PreCombineFn/_PostCombineFn. + Uniform distribution regardless of bundle count. One extra GBK vs NONE, + but distributes streaming state I/O across shard keys. Best for single + global key at high throughput. + """ + NONE = 'none' + HOTKEY_FANOUT = 'hotkey_fanout' + SHARDED = 'sharded' + + +@dataclasses.dataclass(frozen=True) +class WindowSpec: + """Window configuration for metric aggregation. + + Args: + type: FIXED or SLIDING window. + size_seconds: Window size in seconds. + period_seconds: Slide period in seconds (required for SLIDING, ignored for + FIXED). + """ + type: WindowType = WindowType.FIXED + size_seconds: int = 3600 + period_seconds: Optional[int] = None + + +@dataclasses.dataclass(frozen=True) +class DerivedField: + """Pre-aggregation column derivation via expression. + + Args: + name: Name of the new field to create. + expression: A compiled ``Expr`` callable, e.g. + ``Expr.from_string("1 if status == 'success' else 0")``. + """ + name: str + expression: Expr + + +@dataclasses.dataclass(frozen=True) +class MeasureSpec: + """A single aggregation measure. + + Args: + field: Input field name to aggregate. + agg: The aggregation operator. + alias: Output name for this measure's result. + """ + field: str + agg: AggOp + alias: str + + +@dataclasses.dataclass(frozen=True) +class AggregationSpec: + """Windowed grouped aggregation configuration. + + Args: + window: Window configuration. + group_by: Field names for grouping. Empty list means global aggregation. + measures: List of aggregation measures. + """ + window: WindowSpec = dataclasses.field(default_factory=WindowSpec) + group_by: list = dataclasses.field(default_factory=list) + measures: list = dataclasses.field(default_factory=list) + + +@specifiable +class MetricSpec: + """Complete metric computation specification. + + Defines how to transform raw row dicts into a single numeric metric value + suitable for anomaly detection. + + Args: + aggregation: Windowed grouped aggregation spec. + derived_fields: Optional pre-aggregation derived fields. + measure_combiner: Optional post-aggregation ``Expr`` operating on measure + aliases. Required when there are multiple measures. + name: Optional human-readable metric name. + """ + def __init__( + self, + aggregation, + derived_fields=None, + measure_combiner=None, + name=None, + ): + self.name = name + self.aggregation = aggregation + self.derived_fields = derived_fields or [] + self.measure_combiner = measure_combiner + self._validate() + + def _validate(self): + agg = self.aggregation + if not agg.measures: + raise ValueError("MetricSpec requires at least one measure") + if self.measure_combiner is None and len(agg.measures) > 1: + raise ValueError( + "measure_combiner is required when there are multiple measures. " + f"Got {len(agg.measures)} measures: " + f"{[m.alias for m in agg.measures]}") + if (agg.window.type == WindowType.SLIDING and + agg.window.period_seconds is None): + raise ValueError("period_seconds is required for SLIDING windows") + for df in self.derived_fields: + if not isinstance(df.expression, Expr): + raise TypeError( + f"DerivedField.expression must be an Expr, " + f"got {type(df.expression).__name__}") + if (self.measure_combiner is not None and + not isinstance(self.measure_combiner, Expr)): + raise TypeError( + f"measure_combiner must be an Expr, " + f"got {type(self.measure_combiner).__name__}") + # Validate that measure_combiner only references known measure aliases. + if self.measure_combiner is not None: + aliases = {m.alias for m in agg.measures} + unknown = self.measure_combiner.field_refs() - aliases + if unknown: + raise ValueError( + f"measure_combiner references unknown fields: {unknown}. " + f"Available measure aliases: {aliases}") + + def required_source_columns(self): + """Return the set of source table columns needed by this metric spec. + + This includes group_by fields, measure fields (excluding derived field + names), and field references from derived field expressions. + """ + derived_names = {df.name for df in self.derived_fields} + cols = set() + cols.update(self.aggregation.group_by) + for m in self.aggregation.measures: + if m.agg != AggOp.COUNT and m.field not in derived_names: + cols.add(m.field) + for df in self.derived_fields: + cols.update(df.expression.field_refs()) + return cols + + def to_dict(self): + """Serialize to a plain dict suitable for JSON.""" + result = { + 'aggregation': { + 'window': { + 'type': self.aggregation.window.type.value, + 'size_seconds': self.aggregation.window.size_seconds, + 'period_seconds': self.aggregation.window.period_seconds, + }, + 'group_by': list(self.aggregation.group_by), + 'measures': [{ + 'field': m.field, 'agg': m.agg.value, 'alias': m.alias + } for m in self.aggregation.measures], + }, + } + if self.derived_fields: + result['derived_fields'] = [{ + 'name': df.name, 'expression': str(df.expression) + } for df in self.derived_fields] + if self.measure_combiner is not None: + result['measure_combiner'] = {'expression': str(self.measure_combiner)} + if self.name is not None: + result['name'] = self.name + return result + + @classmethod + def from_dict(cls, d): + """Construct a MetricSpec from a plain dict (e.g., loaded from JSON). + + Expressions (``measure_combiner`` and ``derived_fields[].expression``) + are Python expression strings, e.g.:: + + "measure_combiner": {"expression": "clicks / impressions"} + "expression": "1 if status == 'success' else 0" + + Args: + d: Dictionary with keys matching the MetricSpec constructor. + + Returns: + MetricSpec instance. + + Raises: + TypeError: If an expression is not a string. + SyntaxError: If an expression string is not valid Python syntax. + ValueError: If an expression uses unsupported constructs, or if + measure_combiner references fields not in the measure aliases. + """ + agg_dict = d['aggregation'] + window_dict = agg_dict.get('window', {}) + window = WindowSpec( + type=WindowType(window_dict.get('type', 'fixed')), + size_seconds=window_dict.get('size_seconds', 3600), + period_seconds=window_dict.get('period_seconds'), + ) + measures = [ + MeasureSpec(field=m['field'], agg=AggOp(m['agg']), alias=m['alias']) + for m in agg_dict.get('measures', []) + ] + derived_fields = None + if 'derived_fields' in d and d['derived_fields']: + derived_fields = [] + for df in d['derived_fields']: + expr_val = df['expression'] + if not isinstance(expr_val, str): + raise TypeError( + f"derived_fields[].expression must be a string, " + f"got {type(expr_val).__name__}. " + f"Example: \"1 if status == 'success' else 0\"") + derived_fields.append( + DerivedField( + name=df['name'], expression=Expr.from_string(expr_val))) + measure_combiner = None + if 'measure_combiner' in d and d['measure_combiner'] is not None: + mc = d['measure_combiner'] + expr_val = mc['expression'] if isinstance(mc, dict) else mc + if not isinstance(expr_val, str): + raise TypeError( + f"measure_combiner.expression must be a string, " + f"got {type(expr_val).__name__}. " + f"Example: \"clicks / impressions\"") + measure_combiner = Expr.from_string(expr_val) + return cls( + aggregation=AggregationSpec( + window=window, + group_by=agg_dict.get('group_by', []), + measures=measures, + ), + derived_fields=derived_fields, + measure_combiner=measure_combiner, + name=d.get('name'), + _run_init=True, + ) + + +# --------------------------------------------------------------------------- +# Internal CombineFn and DoFns +# --------------------------------------------------------------------------- + + +class _SumCombineFn(beam.CombineFn): + def create_accumulator(self): + return 0 + + def add_input(self, accumulator, element): + return accumulator + element + + def merge_accumulators(self, accumulators): + return sum(accumulators) + + def extract_output(self, accumulator): + return accumulator + + +class _MinCombineFn(beam.CombineFn): + def create_accumulator(self): + return float('inf') + + def add_input(self, accumulator, element): + return element if element < accumulator else accumulator + + def merge_accumulators(self, accumulators): + return min(accumulators) + + def extract_output(self, accumulator): + return accumulator + + +class _MaxCombineFn(beam.CombineFn): + def create_accumulator(self): + return float('-inf') + + def add_input(self, accumulator, element): + return element if element > accumulator else accumulator + + def merge_accumulators(self, accumulators): + return max(accumulators) + + def extract_output(self, accumulator): + return accumulator + + +def _get_combiner_for_agg(agg_op): + """Map AggOp enum to a Beam CombineFn instance.""" + if agg_op == AggOp.SUM: + return _SumCombineFn() + elif agg_op == AggOp.COUNT: + return combiners.CountCombineFn() + elif agg_op == AggOp.MIN: + return _MinCombineFn() + elif agg_op == AggOp.MAX: + return _MaxCombineFn() + elif agg_op == AggOp.MEAN: + return combiners.MeanCombineFn() + else: + raise ValueError(f"Unknown aggregation operator: {agg_op}") + + +class _PreCombineFn(beam.CombineFn): + """Stage 1 wrapper: extract_output returns the raw accumulator.""" + def __init__(self, combine_fn): + self._combine_fn = combine_fn + + def create_accumulator(self): + return self._combine_fn.create_accumulator() + + def add_input(self, accumulator, element): + return self._combine_fn.add_input(accumulator, element) + + def merge_accumulators(self, accumulators): + return self._combine_fn.merge_accumulators(accumulators) + + def extract_output(self, accumulator): + return accumulator # pass raw accumulator, NOT final output + + +class _PostCombineFn(beam.CombineFn): + """Stage 2 wrapper: add_input merges an accumulator from Stage 1.""" + def __init__(self, combine_fn): + self._combine_fn = combine_fn + + def create_accumulator(self): + return self._combine_fn.create_accumulator() + + def add_input(self, accumulator, element): + return self._combine_fn.merge_accumulators([accumulator, element]) + + def merge_accumulators(self, accumulators): + return self._combine_fn.merge_accumulators(accumulators) + + def extract_output(self, accumulator): + return self._combine_fn.extract_output(accumulator) + + +class _MapperSidePrecombine(beam.DoFn): + """Mapper-side pre-aggregation within each bundle. + + Mirrors PGBKCVOperation (operations.py) but as a user-level DoFn so + Dataflow still sees the downstream CombinePerKey and applies incremental + state-side merging. + + Buffers KV elements in an in-memory table keyed by + (window, key), folding values via the CombineFn. On finish_bundle, + emits KV with correct window and timestamp metadata. + + The downstream CombinePerKey must use _PostCombineFn to treat incoming + elements as accumulators. + """ + def __init__(self, combine_fn, max_keys=100_000): + self._combine_fn = combine_fn + self._max_keys = max_keys + + def setup(self): + self._combine_fn.setup() + # Cache bound methods to avoid attribute lookup per element. + self._add_input = self._combine_fn.add_input + self._create_accumulator = self._combine_fn.create_accumulator + compact_method = getattr(self._combine_fn, 'compact', None) + if (compact_method is not None and + compact_method.__func__ is not beam.CombineFn.compact): + self._compact = compact_method + else: + self._compact = None + + def start_bundle(self): + self._table = {} # (window, key) -> [accumulator, timestamp] + self._key_count = 0 + + def process(self, element, window=beam.DoFn.WindowParam, + timestamp=beam.DoFn.TimestampParam): + key, value = element + wkey = (window, key) + table = self._table + entry = table.get(wkey) + if entry is not None: + entry[0] = self._add_input(entry[0], value) + if timestamp < entry[1]: + entry[1] = timestamp + else: + if self._key_count >= self._max_keys: + # Evict 10% of entries, same strategy as PGBKCVOperation. + target = self._key_count * 9 // 10 + old_wkeys = [] + for old_wkey, old_entry in table.items(): + old_wkeys.append(old_wkey) + yield self._make_windowed_value(old_wkey, old_entry) + self._key_count -= 1 + if self._key_count <= target: + break + for old_wkey in reversed(old_wkeys): + del table[old_wkey] + self._key_count += 1 + acc = self._add_input(self._create_accumulator(), value) + table[wkey] = [acc, timestamp] + + def finish_bundle(self): + for wkey, entry in self._table.items(): + yield self._make_windowed_value(wkey, entry) + self._table = None + self._key_count = 0 + + def _make_windowed_value(self, wkey, entry): + window, key = wkey + accumulator, timestamp = entry + if self._compact is not None: + accumulator = self._compact(accumulator) + return beam.utils.windowed_value.WindowedValue( + (key, accumulator), timestamp, (window,)) + + def teardown(self): + self._combine_fn.teardown() + + +class _DerivedFieldsFn: + """Callable that evaluates derived field expressions on each row dict. + + Each derived field's ``expression`` is a compiled ``Expr`` callable. + This class is passed to ``beam.Map`` and is pickle-safe because ``Expr`` + implements ``__reduce__``. + """ + def __init__(self, derived_fields): + self._fields = [(df.name, df.expression) for df in derived_fields] + + def __call__(self, row): + row = dict(row) + for name, expr in self._fields: + row[name] = expr(row) + return row + + +class _ApplyMetricExpr(beam.DoFn): + """DoFn that evaluates a post-aggregation expression on combined results.""" + def __init__(self, measure_combiner, is_keyed): + self._measure_combiner = measure_combiner + self._is_keyed = is_keyed + + def process(self, element, window=beam.DoFn.WindowParam): + if self._is_keyed: + key, agg_dict = element + else: + agg_dict = element + + if self._measure_combiner is not None: + value = float(self._measure_combiner(agg_dict)) + else: + value = float(next(iter(agg_dict.values()))) + + row = beam.Row( + value=value, + window_start=float(window.start), + window_end=float(window.end)) + + if self._is_keyed: + yield (key, row) + else: + yield row + + +class ComputeMetric(beam.PTransform): + """Transforms raw row dicts into metric beam.Rows for anomaly detection. + + Takes a ``PCollection[dict]`` with event-time timestamps and produces + either ``PCollection[beam.Row]`` (for global aggregation) or + ``PCollection[tuple[key, beam.Row]]`` (for grouped aggregation). + + The output is directly compatible with ``AnomalyDetection``. + + Args: + metric_spec: A ``MetricSpec`` defining the metric computation. + fanout_strategy: Strategy for global (non-keyed) aggregation parallelism. + Ignored when group_by is set. Default: SHARDED. + fanout: Number of shards for SHARDED or HOTKEY_FANOUT strategies. + Ignored for NONE. Default: 400. + mapper_side_precombine: If True, pre-aggregates values per key within + each bundle before shuffle (like PGBKCVOperation). Reduces shuffle + volume without adding extra GBKs. The downstream CombinePerKey is + preserved so Dataflow still applies incremental state-side merging. + Effective when few keys see high throughput (>100k rows/sec/key). + """ + def __init__(self, metric_spec, fanout_strategy=FanoutStrategy.SHARDED, + fanout=400, mapper_side_precombine=False): + super().__init__() + self._spec = metric_spec + self._fanout_strategy = fanout_strategy + self._fanout = fanout + self._mapper_side_precombine = mapper_side_precombine + + def expand(self, pcoll): + spec = self._spec + agg = spec.aggregation + + # Step 1: Apply derived fields + if spec.derived_fields: + pcoll = pcoll | 'DerivedFields' >> beam.Map( + _DerivedFieldsFn(spec.derived_fields)) + + # Step 2: Apply windowing + if agg.window.type == WindowType.FIXED: + window_fn = beam_window.FixedWindows(agg.window.size_seconds) + elif agg.window.type == WindowType.SLIDING: + window_fn = beam_window.SlidingWindows( + agg.window.size_seconds, agg.window.period_seconds) + else: + raise ValueError(f"Unknown window type: {agg.window.type}") + + windowed = pcoll | 'Window' >> beam.WindowInto(window_fn) + + # Step 3: Aggregate + measures = agg.measures + aliases = [m.alias for m in measures] + is_keyed = bool(agg.group_by) + + # Single-measure optimization: skip TupleCombineFn overhead (avoids + # tuple creation/unpacking per element on the hot path). + if len(measures) == 1: + combine_fn = _get_combiner_for_agg(measures[0].agg) + _m0 = measures[0] + _a0 = aliases[0] + + def extract_fields(row_dict): + return row_dict.get(_m0.field) if _m0.agg != AggOp.COUNT else 1 + + def to_alias_dict(value): + return {_a0: value} + else: + combine_fn = combiners.TupleCombineFn( + *[_get_combiner_for_agg(m.agg) for m in measures]) + + def extract_fields(row_dict): + return tuple( + row_dict.get(m.field) if m.agg != AggOp.COUNT else 1 + for m in measures) + + def to_alias_dict(values): + return dict(zip(aliases, values)) + + precombine = self._mapper_side_precombine + post_fn = _PostCombineFn(combine_fn) if precombine else None + + if is_keyed: + group_by_fields = agg.group_by + + def extract_key_and_fields(row_dict): + key = tuple(row_dict.get(f) for f in group_by_fields) + return (key, extract_fields(row_dict)) + + keyed = windowed | 'ExtractKey' >> beam.Map(extract_key_and_fields) + if precombine: + aggregated = ( + keyed + | 'Precombine' >> beam.ParDo( + _MapperSidePrecombine(combine_fn)) + | 'Combine' >> beam.CombinePerKey(post_fn) + | 'ToDict' >> beam.MapTuple(lambda k, v: (k, to_alias_dict(v)))) + else: + aggregated = ( + keyed + | 'Combine' >> beam.CombinePerKey(combine_fn) + | 'ToDict' >> beam.MapTuple(lambda k, v: (k, to_alias_dict(v)))) + else: + strategy = self._fanout_strategy + if strategy == FanoutStrategy.NONE: + if precombine: + aggregated = ( + windowed + | 'ExtractFields' >> beam.Map( + lambda row_dict: (None, extract_fields(row_dict))) + | 'Precombine' >> beam.ParDo( + _MapperSidePrecombine(combine_fn)) + | 'DropKey' >> beam.Values() + | 'Combine' >> beam.CombineGlobally( + post_fn).without_defaults() + | 'ToDict' >> beam.Map(to_alias_dict)) + else: + aggregated = ( + windowed + | 'ExtractFields' >> beam.Map(extract_fields) + | 'Combine' >> beam.CombineGlobally( + combine_fn).without_defaults() + | 'ToDict' >> beam.Map(to_alias_dict)) + elif strategy == FanoutStrategy.HOTKEY_FANOUT: + if precombine: + aggregated = ( + windowed + | 'ExtractFields' >> beam.Map( + lambda row_dict: (None, extract_fields(row_dict))) + | 'Precombine' >> beam.ParDo( + _MapperSidePrecombine(combine_fn)) + | 'DropKey' >> beam.Values() + | 'Combine' >> beam.CombineGlobally( + post_fn).with_fanout(self._fanout).without_defaults() + | 'ToDict' >> beam.Map(to_alias_dict)) + else: + aggregated = ( + windowed + | 'ExtractFields' >> beam.Map(extract_fields) + | 'Combine' >> beam.CombineGlobally( + combine_fn).with_fanout(self._fanout).without_defaults() + | 'ToDict' >> beam.Map(to_alias_dict)) + elif strategy == FanoutStrategy.SHARDED: + _num_shards = self._fanout + + def _shard_fields(row_dict): + return (random.randint(0, _num_shards - 1), + extract_fields(row_dict)) + + pre_fn = _PreCombineFn(combine_fn) + shard_post_fn = _PostCombineFn(combine_fn) + if precombine: + aggregated = ( + windowed + | 'ShardAndExtract' >> beam.Map(_shard_fields) + | 'Precombine' >> beam.ParDo( + _MapperSidePrecombine(combine_fn)) + | 'PartialCombine' >> beam.CombinePerKey(post_fn) + | 'DropShard' >> beam.Values() + | 'FinalCombine' >> beam.CombineGlobally( + shard_post_fn).without_defaults() + | 'ToDict' >> beam.Map(to_alias_dict)) + else: + aggregated = ( + windowed + | 'ShardAndExtract' >> beam.Map(_shard_fields) + | 'PartialCombine' >> beam.CombinePerKey(pre_fn) + | 'DropShard' >> beam.Values() + | 'FinalCombine' >> beam.CombineGlobally( + shard_post_fn).without_defaults() + | 'ToDict' >> beam.Map(to_alias_dict)) + else: + raise ValueError(f"Unknown fanout strategy: {strategy}") + + # Step 4: Apply metric expression and set output type hints + metric_dofn = _ApplyMetricExpr(spec.measure_combiner, is_keyed) + + if is_keyed: + # AnomalyDetection checks isinstance(element_type, TupleConstraint) + # to detect keyed input. We must annotate the output type. + result = aggregated | 'MetricExpr' >> beam.ParDo( + metric_dofn).with_output_types(Tuple[Any, beam.Row]) + else: + result = aggregated | 'MetricExpr' >> beam.ParDo( + metric_dofn).with_output_types(beam.Row) + + return result diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py new file mode 100644 index 0000000000..30a61b690e --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py @@ -0,0 +1,1087 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Anomaly monitoring pipeline for BigQuery tables. + +Reads streaming CDC data from BigQuery, computes a configurable windowed +metric, runs anomaly detection, and publishes anomalies to Pub/Sub. + +Designed to be run as a Dataflow Flex Template or locally with DirectRunner. + +Usage (Flex Template):: + + gcloud dataflow flex-template run "sales-monitor-$(date +%Y%m%d)" \\ + --template-file-gcs-location "gs://bucket/anomaly_monitor.json" \\ + --parameters table="project:dataset.table" \\ + --parameters metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' \\ + --parameters detector_spec='{"type":"ZScore"}' \\ + --region us-central1 + +Usage (PrismRunner):: + + python main.py \\ + --table=project:dataset.table \\ + --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' \\ + --detector_spec='{"type":"ZScore"}' \\ + --runner=PrismRunner + +Usage (DataflowRunner):: + + python main.py \\ + --table=project:dataset.table \\ + --metric_spec='' \\ + --detector_spec='' \\ + --runner=DataflowRunner \\ + --project=my-project \\ + --region=us-central1 \\ + --temp_location=gs://bucket/temp \\ + --staging_location=gs://bucket/staging \\ + --setup_file=./setup.py + + +metric_spec JSON Reference +========================== + +Top-level ``metric_spec`` object:: + + { + "aggregation": { ... }, # required + "derived_fields": [ ... ], # optional, pre-aggregation + "measure_combiner": { ... } # optional (required if >1 measure) + } + +aggregation +----------- +:: + + "aggregation": { + "window": { + "type": "fixed" | "sliding", + "size_seconds": , # window size in seconds + "period_seconds": # slide period (required for sliding) + }, + "group_by": ["field1", "field2"], # optional, omit for global agg + "measures": [ + {"field": "", "agg": "", "alias": ""}, + ... + ] + } + +Aggregation operators (``agg``): ``SUM``, ``COUNT``, ``MIN``, ``MAX``, ``MEAN``. + +For ``COUNT``, the ``field`` value is ignored — it counts all rows in the +group. + +Expressions +----------- +Both ``measure_combiner.expression`` and ``derived_fields[].expression`` +are Python expression strings. Bare names are field references, and the +following syntax is supported: + +- Arithmetic: ``+``, ``-``, ``*``, ``/``, ``//``, ``%``, ``**`` +- Comparisons: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=`` +- Boolean logic: ``and``, ``or``, ``not`` +- Negation: ``-field`` +- Conditional: ``true_val if condition else false_val`` +- Functions: ``abs()``, ``min()``, ``max()``, ``round()`` +- Grouping: parentheses for precedence + +``measure_combiner`` references measure aliases and is validated at +pipeline construction time. + +derived_fields +-------------- +Computed before aggregation. Each entry creates a new column available to +measures:: + + "derived_fields": [ + {"name": "is_success", "expression": "1 if status == 'success' else 0"} + ] + +measure_combiner +---------------- +Post-aggregation expression that combines measure aliases into a single +value. Required when there are multiple measures (e.g., ratio metrics):: + + "measure_combiner": {"expression": "clicks / impressions"} + "measure_combiner": {"expression": "(successes + partial) / total"} + + +detector_spec JSON Reference +============================= + +Top-level ``detector_spec`` object:: + + {"type": "", "config": { ... }} + +The ``type`` must be a registered ``@specifiable`` detector class name. +``config`` keys map to that class's ``__init__`` parameters plus inherited +``AnomalyDetector`` parameters. + +Common AnomalyDetector parameters (all detectors):: + + "config": { + "threshold_criterion": { ... }, # optional, see below + "model_id": "" # optional detector ID + } + +``features`` is automatically set to ``['value']`` to match +``ComputeMetric`` output; it does not need to be specified. + +window_size +----------- +All detectors maintain an internal sliding window of recent values for their +statistical trackers (mean, stdev, quantiles, etc.). The default is 1000 +data points. Use ``window_size`` as a shorthand to override this for all +internal trackers at once:: + + {"type": "ZScore", "config": {"window_size": 500}} + +Available detectors +------------------- + +**ZScore** — ``|value - mean| / stdev`` (default threshold: 3):: + + {"type": "ZScore"} + +**IQR** — Interquartile Range (default threshold: 1.5):: + + {"type": "IQR"} + +**RobustZScore** — Modified Z-Score using median/MAD (default threshold: 3.5):: + + {"type": "RobustZScore"} + +threshold_criterion +------------------- +Override the default threshold by nesting a specifiable threshold object. + +**FixedThreshold** — static cutoff (scores >= cutoff are outliers):: + + "threshold_criterion": { + "type": "FixedThreshold", + "config": {"cutoff": 10} + } + +**QuantileThreshold** — dynamic cutoff at a quantile of observed scores:: + + "threshold_criterion": { + "type": "QuantileThreshold", + "config": {"quantile": 0.95} + } + +Both accept optional ``normal_label`` (default 0), ``outlier_label`` +(default 1), and ``missing_label`` (default -2). + +**Threshold** — fixed threshold alert based on a boolean expression. +No warmup period, no history buffer. Alerts whenever the expression +evaluates to true:: + + {"type": "Threshold", "expression": "value >= 0.5"} + {"type": "Threshold", "expression": "value > 100 or value < -100"} + {"type": "Threshold", "expression": "value <= 0.01"} + +The expression receives the computed metric as ``value`` and supports +all safe expression operators (see Expressions section above). + + +Examples +-------- + +Simple SUM metric with ZScore:: + + --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":3600},"measures":[{"field":"transaction_amount","agg":"SUM","alias":"revenue"}]}}' + --detector_spec='{"type":"ZScore"}' + +Grouped ratio metric (CTR) with ZScore:: + + --metric_spec='{"aggregation":{"window":{"type":"fixed","size_seconds":10},"group_by":["campaign_type","browser_version"],"measures":[{"field":"is_click","agg":"SUM","alias":"clicks"},{"field":"is_click","agg":"COUNT","alias":"impressions"}]},"measure_combiner":{"expression":"clicks / impressions"}}' + --detector_spec='{"type":"ZScore"}' + +Derived field + ratio + custom threshold:: + + --metric_spec='{"derived_fields":[{"name":"is_success","expression":"1 if status == \\'success\\' else 0"}],"aggregation":{"window":{"type":"fixed","size_seconds":10},"group_by":["brand_name","category"],"measures":[{"field":"is_success","agg":"SUM","alias":"successes"},{"field":"is_success","agg":"COUNT","alias":"total"}]},"measure_combiner":{"expression":"successes / total"}}' + --detector_spec='{"type":"ZScore","config":{"threshold_criterion":{"type":"FixedThreshold","config":{"cutoff":10}}}}' +""" + +import datetime +import json +import logging +import re +import time + +import apache_beam as beam +from apache_beam.io.gcp.bigquery import WriteToBigQuery +from apache_beam.io.gcp.pubsub import WriteToPubSub +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.options.pipeline_options import SetupOptions + +from bqmonitor.metric import ComputeMetric +from bqmonitor.metric import FanoutStrategy +from bqmonitor.metric import MetricSpec +from bqmonitor.safe_eval import Expr +from apache_beam.ml.anomaly.base import AnomalyPrediction +from apache_beam.ml.anomaly.base import AnomalyResult +from apache_beam.ml.anomaly.specifiable import Spec +from apache_beam.ml.anomaly.specifiable import Specifiable +from apache_beam.ml.anomaly.transforms import AnomalyDetection + +# Import detectors so they register with @specifiable before from_spec. +from apache_beam.ml.anomaly.detectors import zscore # noqa: F401 +from apache_beam.ml.anomaly.detectors import iqr # noqa: F401 +from apache_beam.ml.anomaly.detectors import robust_zscore # noqa: F401 + +_LOGGER = logging.getLogger(__name__) + +_SUPPORTED_DETECTORS = ('ZScore', 'IQR', 'RobustZScore') + +# Matches project:dataset.table or project.dataset.table +_TABLE_RE = re.compile( + r'^[a-zA-Z0-9][a-zA-Z0-9_-]*[:\.][a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$') + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _validate_topic_path(topic): + """Validate that a Pub/Sub topic is a full resource path. + + Args: + topic: Full Pub/Sub topic path, e.g. + 'projects/my-project/topics/my-topic'. + + Returns: + The validated topic path. + + Raises: + ValueError: If the topic is not a full resource path. + """ + if not (topic.startswith('projects/') and '/topics/' in topic): + raise ValueError( + f"--topic must be a full Pub/Sub resource path " + f"(projects//topics/), got: '{topic}'") + return topic + + +def _unpack_result(element): + """Unpack a possibly-keyed AnomalyResult element. + + Returns: + (key, result) where key is None for unkeyed elements. + """ + if isinstance(element, tuple) and len(element) == 2: + return element[0], element[1] + return None, element + + +def _parse_table_ref(table): + """Parse and validate a table reference string. + + Args: + table: Table reference in 'project:dataset.table' or + 'project.dataset.table' format. + + Returns: + (project, dataset, table_name) tuple. + + Raises: + ValueError: If the table string doesn't match the expected format. + """ + if not _TABLE_RE.match(table): + raise ValueError( + f"Invalid --table format: '{table}'. " + f"Expected: project:dataset.table or project.dataset.table") + if ':' in table: + project, rest = table.split(':', 1) + dataset, table_name = rest.split('.', 1) + else: + project, dataset, table_name = table.split('.', 2) + return project, dataset, table_name + + +# --------------------------------------------------------------------------- +# DoFns +# --------------------------------------------------------------------------- + + +class _LogAnomalyResult(beam.DoFn): + """Logs each AnomalyResult at WARNING level for visibility in Dataflow.""" + def process(self, element): + key, result = _unpack_result(element) + prediction = result.predictions[0] + example = result.example + + if prediction.label == 1: + tag = '!! OUTLIER !!' + elif prediction.label == 0: + tag = 'NORMAL' + else: + tag = 'WARMUP' + + ws = datetime.datetime.fromtimestamp( + example.window_start, tz=datetime.timezone.utc).strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + we = datetime.datetime.fromtimestamp( + example.window_end, tz=datetime.timezone.utc).strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + window_str = f'{ws}-{we}' + + if key is not None: + _LOGGER.warning( + '[%s] window=%s key=%s value=%.2f score=%s label=%s', + tag, window_str, key, example.value, prediction.score, + prediction.label) + else: + _LOGGER.warning( + '[%s] window=%s value=%.2f score=%s label=%s', + tag, window_str, example.value, prediction.score, + prediction.label) + yield element + + +class _ThresholdAlert(beam.DoFn): + """Evaluates a threshold expression against metric values. + + Emits AnomalyResult elements consistent with the statistical detectors, + allowing threshold alerts to flow through the same logging and Pub/Sub + pipeline. + + The expression is evaluated with ``value`` bound to the metric value. + If it evaluates to truthy, the element is labelled as an outlier (1); + otherwise it is labelled normal (0). + + Example expressions: ``value >= 0.5``, ``value <= 0.01``, + ``value > 100 or value < -100``. + """ + + def __init__(self, expression_text): + self._expression_text = expression_text + self._expr = None + + def setup(self): + self._expr = Expr(self._expression_text) + + def process(self, element): + if isinstance(element, tuple) and len(element) == 2: + key, row = element + else: + key, row = None, element + + value = row.value + is_alert = bool(self._expr({'value': value})) + + prediction = AnomalyPrediction( + model_id=f'Threshold({self._expression_text})', + score=None, + label=1 if is_alert else 0, + threshold=None) + + result = AnomalyResult(example=row, predictions=[prediction]) + + if key is not None: + yield (key, result) + else: + yield result + + +class _FormatAnomalyAsJson(beam.DoFn): + """Converts anomaly results (label == 1) to JSON byte strings for Pub/Sub.""" + def process(self, element): + key, result = _unpack_result(element) + prediction = result.predictions[0] + if prediction.label != 1: + return + + example = result.example + ws = datetime.datetime.fromtimestamp( + example.window_start, tz=datetime.timezone.utc).strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + we = datetime.datetime.fromtimestamp( + example.window_end, tz=datetime.timezone.utc).strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + + payload = { + 'event_description': ( + f'Anomaly detected value={example.value}' + f' score={prediction.score}' + f' in window={ws}-{we}'), + 'agent_id': prediction.model_id, + } + if key is not None: + payload['key'] = str(key) + + yield json.dumps(payload).encode('utf-8') + + +_SINK_SCHEMA = { + 'fields': [ + {'name': 'window_start', 'type': 'TIMESTAMP', 'mode': 'REQUIRED'}, + {'name': 'window_end', 'type': 'TIMESTAMP', 'mode': 'REQUIRED'}, + {'name': 'value', 'type': 'FLOAT64', 'mode': 'REQUIRED'}, + {'name': 'score', 'type': 'FLOAT64', 'mode': 'NULLABLE'}, + {'name': 'label', 'type': 'INT64', 'mode': 'REQUIRED'}, + {'name': 'key', 'type': 'STRING', 'mode': 'NULLABLE'}, + ] +} + + +class _FormatResultForBQ(beam.DoFn): + """Converts all AnomalyResult elements to BQ row dicts.""" + def process(self, element): + key, result = _unpack_result(element) + prediction = result.predictions[0] + example = result.example + + row = { + 'window_start': datetime.datetime.fromtimestamp( + example.window_start, tz=datetime.timezone.utc).isoformat(), + 'window_end': datetime.datetime.fromtimestamp( + example.window_end, tz=datetime.timezone.utc).isoformat(), + 'value': float(example.value), + 'score': float(prediction.score) if prediction.score is not None + else None, + 'label': int(prediction.label), + } + if key is not None: + row['key'] = str(key) + + yield row + + +# --------------------------------------------------------------------------- +# Pipeline options +# --------------------------------------------------------------------------- + + +class AnomalyMonitorOptions(PipelineOptions): + """Pipeline options for the anomaly monitor.""" + @classmethod + def _add_argparse_args(cls, parser): + parser.add_argument( + '--table', + default=None, + help='BigQuery table to monitor. ' + 'Format: project:dataset.table') + parser.add_argument( + '--metric_spec', + default=None, + help='JSON string defining the metric computation. ' + 'See MetricSpec.from_dict() for schema.') + parser.add_argument( + '--detector_spec', + default=None, + help='JSON string defining the anomaly detector. ' + 'Format: {"type":"ZScore"} or ' + '{"type":"ZScore","config":{"threshold_criterion":{...}}}') + parser.add_argument( + '--poll_interval_sec', + type=int, + default=60, + help='Seconds between BigQuery CDC polls. Default 60.') + parser.add_argument( + '--change_function', + default='APPENDS', + choices=['APPENDS', 'CHANGES'], + help='BigQuery change function to use. Default APPENDS.') + parser.add_argument( + '--buffer_sec', + type=float, + default=15.0, + help='Safety buffer behind now() in seconds. Default 15.') + parser.add_argument( + '--start_offset_sec', + type=float, + default=60.0, + help='Start reading from this many seconds ago. Default 60.') + parser.add_argument( + '--duration_sec', + type=float, + default=0.0, + help='How long to run in seconds. 0 means run forever. Default 0.') + parser.add_argument( + '--temp_dataset', + default=None, + help='BigQuery dataset for temp tables. If unset, auto-created.') + parser.add_argument( + '--topic', + default=None, + help='Pub/Sub topic for anomaly results. ' + 'Full path: projects//topics/.') + parser.add_argument( + '--log_all_results', + default='false', + help='Log all anomaly detection results (normal, outlier, warmup) ' + 'at WARNING level. Default: false.') + parser.add_argument( + '--sink_table', + default=None, + help='BigQuery table to write all anomaly detection results to. ' + 'Format: project:dataset.table. If unset, results are not written ' + 'to BigQuery.') + parser.add_argument( + '--decompress_shards', + type=int, + default=400, + help='Number of shards for CDC Arrow batch decompression fan-out. ' + 'Spreads decompression CPU across workers. ' + '0 disables fan-out (decode inline). Default: 400.') + parser.add_argument( + '--fanout_strategy', + default='sharded', + choices=['sharded', 'hotkey_fanout', 'none'], + help='Parallelism strategy for global (non-keyed) metric ' + 'aggregation. Ignored when group_by is set. Default: sharded.') + parser.add_argument( + '--fanout', + type=int, + default=400, + help='Number of shards for sharded or hotkey_fanout strategies. ' + 'Ignored for none. Default: 400.') + parser.add_argument( + '--mapper_side_precombine', + type=lambda v: v.lower() in ('true', '1', 'yes'), + default=False, + help='Enable mapper-side pre-aggregation within each bundle before ' + 'shuffle. Reduces shuffle volume by folding values per key locally ' + 'while preserving Dataflow incremental state-side merging. ' + 'Effective at >100k rows/sec/key. Default: false.') + + +# --------------------------------------------------------------------------- +# Spec parsing +# --------------------------------------------------------------------------- + + +def _parse_metric_spec(json_str): + """Parse a MetricSpec from a JSON string. + + Raises: + ValueError: If the JSON is malformed or the spec is invalid. + """ + try: + d = json.loads(json_str) + except json.JSONDecodeError as e: + raise ValueError( + f"Invalid JSON in --metric_spec: {e}. " + f"Value: {json_str[:200]}") from e + try: + return MetricSpec.from_dict(d) + except (ValueError, TypeError, KeyError) as e: + raise ValueError(f"Invalid --metric_spec: {e}") from e + + +def _dict_to_spec(d): + """Recursively convert nested dicts with ``type`` keys into Spec objects. + + ``json.loads`` produces plain dicts, but ``Specifiable.from_spec`` expects + ``Spec`` objects for nested specifiables (e.g. ``threshold_criterion`` + inside a detector config). Without this conversion the nested dict passes + through ``_specifiable_from_spec_helper`` unchanged and the detector + receives a raw dict instead of the expected ``ThresholdFn`` instance. + """ + if isinstance(d, dict) and 'type' in d: + config = d.get('config', {}) + if config: + config = {k: _dict_to_spec(v) for k, v in config.items()} + return Spec(type=d['type'], config=config) + if isinstance(d, list): + return [_dict_to_spec(item) for item in d] + return d + + +def _expand_window_size(d): + """Expand ``window_size`` shorthand into detector-specific tracker configs. + + Instead of constructing deeply nested tracker specs, users can write:: + + {"type": "ZScore", "config": {"window_size": 500}} + + This expands into the full nested tracker configuration that each detector + type expects. If the user already set explicit tracker configs, those take + precedence (``setdefault`` semantics). + + Raises: + ValueError: If window_size is not a positive integer. + """ + config = d.get('config', {}) + ws = config.pop('window_size', None) + if ws is None: + return + + if not isinstance(ws, int) or ws <= 0: + raise ValueError( + f"window_size must be a positive integer, got {ws!r}") + + detector_type = d['type'] + + if detector_type == 'ZScore': + config.setdefault( + 'sub_stat_tracker', + {'type': 'IncSlidingMeanTracker', 'config': {'window_size': ws}}) + config.setdefault( + 'stdev_tracker', + {'type': 'IncSlidingStdevTracker', 'config': {'window_size': ws}}) + elif detector_type == 'IQR': + config.setdefault( + 'q1_tracker', + { + 'type': 'BufferedSlidingQuantileTracker', + 'config': {'window_size': ws, 'q': 0.25} + }) + # q3_tracker auto-derives from q1_tracker in IQR.__init__ + elif detector_type == 'RobustZScore': + _median_tracker_spec = { + 'type': 'MedianTracker', + 'config': { + 'quantile_tracker': { + 'type': 'BufferedSlidingQuantileTracker', + 'config': {'window_size': ws, 'q': 0.5} + } + } + } + config.setdefault( + 'mad_tracker', + { + 'type': 'MadTracker', + 'config': { + 'median_tracker': _median_tracker_spec, + 'diff_median_tracker': { + 'type': 'MedianTracker', + 'config': { + 'quantile_tracker': { + 'type': 'BufferedSlidingQuantileTracker', + 'config': {'window_size': ws, 'q': 0.5} + } + } + } + } + }) + + +def _parse_detector_spec(json_str): + """Parse an anomaly detector from a JSON Spec string. + + The JSON should have the form:: + + {"type": "ZScore"} + + Nested specifiable objects (e.g. ``threshold_criterion``) are supported:: + + {"type": "ZScore", "config": { + "threshold_criterion": {"type": "FixedThreshold", "config": {"cutoff": 10}} + }} + + A ``window_size`` shorthand sets the history buffer for all internal + trackers:: + + {"type": "ZScore", "config": {"window_size": 500}} + + **Threshold** — a simple fixed-threshold alerter that evaluates a boolean + expression against the metric value. No warmup period, no history:: + + {"type": "Threshold", "expression": "value >= 0.5"} + {"type": "Threshold", "expression": "value > 100 or value < -100"} + + The expression may use ``value`` (the computed metric) and all safe + expression operators (see Expressions section above). + + For statistical detectors, the ``type`` field must match a registered + @specifiable detector class (e.g. ZScore, IQR, RobustZScore). + + ``features`` is automatically set to ``['value']`` to match the output of + ``ComputeMetric``. Any user-supplied ``features`` is overwritten. + + Returns: + For statistical detectors: an instantiated AnomalyDetector. + For Threshold: a ``_ThresholdAlert`` DoFn instance. + + Raises: + ValueError: If the JSON is malformed, detector type is unknown, or + the spec is otherwise invalid. + """ + try: + d = json.loads(json_str) + except json.JSONDecodeError as e: + raise ValueError( + f"Invalid JSON in --detector_spec: {e}. " + f"Value: {json_str[:200]}") from e + + if not isinstance(d, dict) or 'type' not in d: + raise ValueError( + "detector_spec must be a JSON object with a 'type' field. " + f"Example: {{\"type\":\"ZScore\"}}. Got: {json_str[:200]}") + + detector_type = d['type'] + + if detector_type == 'Threshold': + expr_text = d.get('expression') + if not expr_text: + raise ValueError( + "Threshold detector requires an 'expression' field. " + "Example: {\"type\":\"Threshold\",\"expression\":\"value >= 0.5\"}") + # Validate the expression at parse time. + try: + expr = Expr(expr_text) + except (ValueError, SyntaxError) as e: + raise ValueError( + f"Invalid threshold expression: {e}") from e + if 'value' not in expr.field_refs(): + _LOGGER.warning( + "Threshold expression '%s' does not reference 'value'. " + "It will receive the computed metric value as 'value'.", expr_text) + return _ThresholdAlert(expr_text) + + if detector_type not in _SUPPORTED_DETECTORS: + raise ValueError( + f"Unknown detector type '{detector_type}'. " + f"Supported detectors: {', '.join(_SUPPORTED_DETECTORS)}, Threshold") + + d.setdefault('config', {}) + d['config']['features'] = ['value'] + _expand_window_size(d) + spec = _dict_to_spec(d) + try: + return Specifiable.from_spec(spec, _run_init=True) + except (ValueError, TypeError) as e: + raise ValueError( + f"Failed to construct {detector_type} detector: {e}") from e + + +# --------------------------------------------------------------------------- +# Preflight checks +# --------------------------------------------------------------------------- + + +def _preflight_checks(options, metric_spec): + """Validate GCP resources are accessible before building the pipeline. + + Checks: + - BigQuery source table exists and is readable. + - Required metric columns exist in the source table (dry-run query). + - BigQuery temp dataset is writable (if specified) or datasets.create + permission exists (dry-run only — does not actually create). + - Pub/Sub topic exists. + + Logs warnings and continues if a check cannot be performed (e.g. missing + client library). Raises ValueError on definite failures. + """ + project, dataset, table_name = _parse_table_ref(options.table) + topic_path = _validate_topic_path(options.topic) + + required_columns = sorted(metric_spec.required_source_columns()) + _check_bq_source_table(project, dataset, table_name, options, + required_columns) + _check_bq_temp_dataset(project, options) + _check_pubsub_topic(topic_path) + + +def _check_bq_source_table(project, dataset, table_name, options, + required_columns): + """Verify the source BigQuery table exists and required columns are present. + + Runs a dry-run CDC query selecting the columns referenced by the metric + spec. This validates table access, CDC function access, and column + existence in a single round-trip. + """ + try: + from apache_beam.io.gcp import bigquery_tools + from apache_beam.io.gcp.internal.clients import bigquery + except ImportError: + _LOGGER.warning( + '[Preflight] Skipping BQ table check: ' + 'BigQuery client libraries not available') + return + + try: + bq = bigquery_tools.BigQueryWrapper() + bq.get_table(project, dataset, table_name) + _LOGGER.info( + '[Preflight] Source table %s:%s.%s is accessible', + project, dataset, table_name) + except Exception as e: + raise ValueError( + f"Cannot access BigQuery table '{project}:{dataset}.{table_name}'. " + f"Verify it exists and the service account has " + f"bigquery.tables.get and bigquery.tables.getData permissions. " + f"Error: {e}") from e + + # Dry-run a CDC query selecting the metric's required columns. + # This validates CDC function access and column existence in one step. + select_clause = ', '.join(required_columns) if required_columns else '1' + try: + sql = ( + f"SELECT {select_clause} FROM {options.change_function}" + f"(TABLE `{project}.{dataset}.{table_name}`, " + f"NULL, NULL) LIMIT 0") + _LOGGER.info('[Preflight] Dry-run query: %s', sql) + request = bigquery.BigqueryJobsInsertRequest( + projectId=project, + job=bigquery.Job( + configuration=bigquery.JobConfiguration( + query=bigquery.JobConfigurationQuery( + query=sql, + useLegacySql=False), + dryRun=True))) + bq.client.jobs.Insert(request) + _LOGGER.info( + '[Preflight] %s() access and columns %s verified for %s:%s.%s', + options.change_function, required_columns, + project, dataset, table_name) + except Exception as e: + raise ValueError( + f"Cannot execute {options.change_function}() on " + f"'{project}:{dataset}.{table_name}' " + f"with columns {required_columns}. " + f"Verify the table has change history enabled, the columns exist, " + f"and the service account has bigquery.jobs.create permission. " + f"Error: {e}") from e + + +def _check_bq_temp_dataset(project, options): + """Verify access to the temp dataset (if specified), or check that + datasets.create permission exists for auto-creation.""" + try: + from apache_beam.io.gcp import bigquery_tools + from apache_beam.io.gcp.internal.clients import bigquery + from apitools.base.py.exceptions import HttpError + except ImportError: + _LOGGER.warning( + '[Preflight] Skipping BQ temp dataset check: ' + 'BigQuery client libraries not available') + return + + if options.temp_dataset: + try: + bq = bigquery_tools.BigQueryWrapper() + bq.client.datasets.Get( + bigquery.BigqueryDatasetsGetRequest( + projectId=project, datasetId=options.temp_dataset)) + _LOGGER.info( + '[Preflight] Temp dataset %s:%s exists', + project, options.temp_dataset) + except HttpError as e: + if e.status_code == 404: + raise ValueError( + f"Temp dataset '{project}:{options.temp_dataset}' not found. " + f"Create it or omit --temp_dataset for auto-creation.") from e + elif e.status_code == 403: + raise ValueError( + f"No access to temp dataset '{project}:{options.temp_dataset}'. " + f"Verify the service account has " + f"bigquery.datasets.get permission.") from e + raise + + # Verify we can write to the temp dataset by doing a dry-run query + # with a destination table in it. + try: + temp_table_ref = bigquery.TableReference( + projectId=project, + datasetId=options.temp_dataset, + tableId='beam_ch_preflight_check') + request = bigquery.BigqueryJobsInsertRequest( + projectId=project, + job=bigquery.Job( + configuration=bigquery.JobConfiguration( + query=bigquery.JobConfigurationQuery( + query='SELECT 1', + useLegacySql=False, + destinationTable=temp_table_ref, + writeDisposition='WRITE_TRUNCATE'), + dryRun=True))) + bq.client.jobs.Insert(request) + _LOGGER.info( + '[Preflight] Write access to temp dataset %s:%s verified', + project, options.temp_dataset) + except Exception as e: + raise ValueError( + f"Cannot write to temp dataset '{project}:{options.temp_dataset}'. " + f"Verify the service account has bigquery.tables.create and " + f"bigquery.tables.updateData permissions on this dataset. " + f"Error: {e}") from e + else: + _LOGGER.info( + '[Preflight] No --temp_dataset specified; ' + 'will auto-create at runtime (requires bigquery.datasets.create)') + + +def _check_pubsub_topic(topic_path): + """Verify the Pub/Sub topic exists.""" + try: + from google.cloud import pubsub_v1 + from google.api_core.exceptions import NotFound, PermissionDenied + except ImportError: + _LOGGER.warning( + '[Preflight] Skipping Pub/Sub check: ' + 'google-cloud-pubsub not available') + return + + try: + publisher = pubsub_v1.PublisherClient() + publisher.get_topic(topic=topic_path) + _LOGGER.info('[Preflight] Pub/Sub topic %s is accessible', topic_path) + except NotFound: + raise ValueError( + f"Pub/Sub topic '{topic_path}' not found. " + f"Create it with: gcloud pubsub topics create {topic_path}") + except PermissionDenied as e: + raise ValueError( + f"No permission to access Pub/Sub topic '{topic_path}'. " + f"Verify the service account has pubsub.topics.get and " + f"pubsub.topics.publish permissions. Error: {e}") from e + except Exception as e: + _LOGGER.warning( + '[Preflight] Could not verify Pub/Sub topic %s: %s', + topic_path, e) + + +# --------------------------------------------------------------------------- +# Pipeline construction +# --------------------------------------------------------------------------- + + +def build_pipeline(pipeline, options, metric_spec, detector): + """Construct the anomaly monitoring pipeline. + + Args: + pipeline: A beam.Pipeline instance. + options: AnomalyMonitorOptions with table, poll_interval_sec, etc. + metric_spec: Parsed MetricSpec instance. + detector: Parsed anomaly detector instance. + + Returns: + The final PCollection (for testing). + """ + from bqmonitor.cdc import ReadBigQueryChangeHistory + + start_time = time.time() - options.start_offset_sec + stop_time = ( + time.time() + options.duration_sec if options.duration_sec > 0 else None) + + _LOGGER.info('Anomaly Monitor Pipeline') + _LOGGER.info(' Table: %s', options.table) + _LOGGER.info(' Detector: %s', type(detector).__name__) + _LOGGER.info(' Poll interval: %d sec', options.poll_interval_sec) + _LOGGER.info(' Change function: %s', options.change_function) + + columns = sorted(metric_spec.required_source_columns()) + _LOGGER.info(' Columns: %s', columns) + + # Auto-rename pseudo-columns if they conflict with user column names. + change_type_col = 'change_type' + change_ts_col = 'change_timestamp' + col_set = set(columns) + if change_type_col in col_set: + change_type_col = '_bqm_change_type' + _LOGGER.info( + ' Renamed pseudo-column change_type -> %s to avoid conflict', + change_type_col) + if change_ts_col in col_set: + change_ts_col = '_bqm_change_timestamp' + _LOGGER.info( + ' Renamed pseudo-column change_timestamp -> %s to avoid conflict', + change_ts_col) + + cdc_kwargs = dict( + table=options.table, + poll_interval_sec=options.poll_interval_sec, + start_time=start_time, + change_function=options.change_function, + buffer_sec=options.buffer_sec, + columns=columns, + change_type_column=change_type_col, + change_timestamp_column=change_ts_col, + decompress_shards=( + options.decompress_shards if options.decompress_shards > 0 + else None)) + if stop_time is not None: + cdc_kwargs['stop_time'] = stop_time + if options.temp_dataset: + cdc_kwargs['temp_dataset'] = options.temp_dataset + + rows = pipeline | 'ReadCDC' >> ReadBigQueryChangeHistory(**cdc_kwargs) + fanout_strategy = FanoutStrategy(options.fanout_strategy) + metrics = rows | 'ComputeMetric' >> ComputeMetric( + metric_spec, fanout_strategy=fanout_strategy, fanout=options.fanout, + mapper_side_precombine=options.mapper_side_precombine) + + # Rewindow into GlobalWindows so the anomaly detector sees the full + # stream of window results as a time series, not isolated per-window. + from apache_beam.transforms.window import GlobalWindows + global_metrics = metrics | 'Rewindow' >> beam.WindowInto(GlobalWindows()) + + if isinstance(detector, _ThresholdAlert): + anomalies = global_metrics | 'DetectAnomalies' >> beam.ParDo(detector) + else: + anomalies = global_metrics | 'DetectAnomalies' >> AnomalyDetection(detector) + + if options.log_all_results.lower() == 'true': + _ = anomalies | 'LogResults' >> beam.ParDo(_LogAnomalyResult()) + + # Publish anomalies (label == 1) to Pub/Sub. + topic_path = _validate_topic_path(options.topic) + + _ = ( + anomalies + | 'FormatAnomalies' >> beam.ParDo(_FormatAnomalyAsJson()) + | 'WriteToPubSub' >> WriteToPubSub(topic=topic_path)) + + # Write all results to a BigQuery sink table (if configured). + if options.sink_table: + sink_table = options.sink_table.replace(':', '.') + _ = ( + anomalies + | 'FormatForBQ' >> beam.ParDo(_FormatResultForBQ()) + | 'WriteSink' >> WriteToBigQuery( + table=sink_table, + method='STREAMING_INSERTS', + schema=_SINK_SCHEMA, + create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, + write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)) + + return anomalies + + +def run(argv=None): + """Main entry point.""" + options = PipelineOptions(argv) + monitor_options = options.view_as(AnomalyMonitorOptions) + + # Validate required options. + for required_opt in ('table', 'metric_spec', 'detector_spec', 'topic'): + if getattr(monitor_options, required_opt) is None: + raise ValueError(f'--{required_opt} is required') + + # Validate table format. + _parse_table_ref(monitor_options.table) + + # Parse specs early so errors surface before pipeline construction. + metric_spec = _parse_metric_spec(monitor_options.metric_spec) + detector = _parse_detector_spec(monitor_options.detector_spec) + + # Check GCP resources are accessible. + _preflight_checks(monitor_options, metric_spec) + + options.view_as(SetupOptions).save_main_session = True + + with beam.Pipeline(options=options) as p: + build_pipeline(p, monitor_options, metric_spec, detector) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py new file mode 100644 index 0000000000..3cc9e00d21 --- /dev/null +++ b/python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/safe_eval.py @@ -0,0 +1,192 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Safe expression evaluator for metric computation. + +Parses a Python expression string, validates it uses only allowed +constructs, compiles it, and produces a callable that evaluates the +expression against a field context dict. + +Example usage:: + + from bqmonitor.safe_eval import Expr + + expr = Expr.from_string("clicks / impressions") + expr({'clicks': 50, 'impressions': 1000}) # 0.05 + + expr = Expr.from_string("1 if status == 'success' else 0") + expr({'status': 'success'}) # 1 + +Allowed constructs: field names (bare names), literals (int, float, str), +arithmetic (``+, -, *, /, //, %, **``), comparisons (``==, !=, <, <=, >, >=``), +boolean logic (``and, or, not``), unary negation, ``if/else``, and +safe builtins (``abs, min, max, round``). +""" + +import ast + +# --- AST whitelist --- + +_ALLOWED_BINOPS = ( + ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow) + +_ALLOWED_CMPOPS = (ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE) + +_SAFE_BUILTINS = {"abs": abs, "min": min, "max": max, "round": round} + + +def _validate_ast(node): + """Recursively validate that an AST node uses only allowed constructs.""" + if isinstance(node, ast.Name): + return + + if isinstance(node, ast.Constant): + if not isinstance(node.value, (int, float, str)): + raise ValueError( + f"Unsupported literal type: {type(node.value).__name__}. " + f"Only int, float, and str literals are supported.") + return + + if isinstance(node, ast.UnaryOp): + if not isinstance(node.op, (ast.USub, ast.Not)): + raise ValueError( + f"Unsupported unary operator: {type(node.op).__name__}. " + f"Only negation (-) and not are supported.") + _validate_ast(node.operand) + return + + if isinstance(node, ast.BinOp): + if not isinstance(node.op, _ALLOWED_BINOPS): + raise ValueError( + f"Unsupported binary operator: {type(node.op).__name__}. " + f"Supported: +, -, *, /, //, %, **") + _validate_ast(node.left) + _validate_ast(node.right) + return + + if isinstance(node, ast.BoolOp): + # and / or + for value in node.values: + _validate_ast(value) + return + + if isinstance(node, ast.Compare): + if len(node.ops) != 1 or len(node.comparators) != 1: + raise ValueError( + "Chained comparisons not supported (e.g., a < b < c). " + "Use (a < b) and separate expressions instead.") + if not isinstance(node.ops[0], _ALLOWED_CMPOPS): + raise ValueError( + f"Unsupported comparison: {type(node.ops[0]).__name__}. " + f"Supported: ==, !=, <, <=, >, >=") + _validate_ast(node.left) + _validate_ast(node.comparators[0]) + return + + if isinstance(node, ast.IfExp): + _validate_ast(node.test) + _validate_ast(node.body) + _validate_ast(node.orelse) + return + + if isinstance(node, ast.Call): + if not (isinstance(node.func, ast.Name) + and node.func.id in _SAFE_BUILTINS): + name = node.func.id if isinstance(node.func, ast.Name) else ast.dump( + node.func) + raise ValueError( + f"Unsupported function: {name}. " + f"Supported: {', '.join(sorted(_SAFE_BUILTINS))}.") + if node.keywords: + raise ValueError("Keyword arguments not supported in function calls.") + for arg in node.args: + _validate_ast(arg) + return + + raise ValueError( + f"Unsupported expression: {ast.dump(node)}. " + f"Only field names, literals, arithmetic (+,-,*,/,//,%,**), " + f"comparisons (==,!=,<,<=,>,>=), boolean logic (and, or, not), " + f"if/else, and functions ({', '.join(sorted(_SAFE_BUILTINS))}) " + f"are supported.") + + +def _collect_field_refs(node): + """Collect all field names referenced in an AST (excluding builtins).""" + return frozenset( + child.id for child in ast.walk(node) + if isinstance(child, ast.Name) and child.id not in _SAFE_BUILTINS) + + +class Expr: + """A validated, compiled expression callable. + + Parses a Python expression string, validates it uses only safe + constructs, and compiles it into a callable. The compiled expression + is evaluated with restricted builtins (no access to ``import``, + ``open``, ``exec``, etc.). + + Args: + text: A Python expression string. + + Raises: + ValueError: If the expression uses unsupported Python constructs. + SyntaxError: If the string is not valid Python syntax. + """ + def __init__(self, text): + self._text = text + tree = ast.parse(text, mode='eval') + _validate_ast(tree.body) + self._code = compile(tree, '', 'eval') + self._refs = _collect_field_refs(tree.body) + + def __call__(self, context): + """Evaluate the expression against a dict of field values.""" + env = dict(_SAFE_BUILTINS) + env.update(context) + return eval(self._code, {"__builtins__": {}}, env) + + def field_refs(self): + """Return the set of field names referenced by this expression.""" + return set(self._refs) + + @staticmethod + def from_string(text): + """Parse a Python expression string into a compiled Expr callable. + + Examples:: + + Expr.from_string("clicks / impressions") + Expr.from_string("1 if status == 'success' else 0") + Expr.from_string("(a + b) / total") + """ + return Expr(text) + + def __str__(self): + return self._text + + def __repr__(self): + return f"Expr({self._text!r})" + + def __eq__(self, other): + return isinstance(other, Expr) and self._text == other._text + + def __hash__(self): + return hash(self._text) + + def __reduce__(self): + """Pickle support: store text, recompile on unpickle.""" + return (Expr, (self._text, )) diff --git a/python/src/test/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetectionIT.java b/python/src/test/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetectionIT.java new file mode 100644 index 0000000000..89d8a6c9ad --- /dev/null +++ b/python/src/test/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetectionIT.java @@ -0,0 +1,594 @@ +/* + * Copyright (C) 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.templates.python; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.TableResult; +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import com.google.common.collect.ImmutableMap; +import com.google.pubsub.v1.ReceivedMessage; +import com.google.pubsub.v1.SubscriptionName; +import com.google.pubsub.v1.TopicName; +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; +import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.PipelineOperator.Result; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.gcp.TemplateTestBase; +import org.apache.beam.it.gcp.bigquery.BigQueryResourceManager; +import org.apache.beam.it.gcp.bigquery.matchers.BigQueryAsserts; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.pubsub.conditions.PubsubMessagesCheck; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration test for {@link BigQueryAnomalyDetection}. + * + *

Inserts normal baseline data into a BQ table, then injects an anomalous spike, and verifies + * that the pipeline detects the anomaly and publishes it to Pub/Sub. + */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(BigQueryAnomalyDetection.class) +@RunWith(JUnit4.class) +public final class BigQueryAnomalyDetectionIT extends TemplateTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(BigQueryAnomalyDetectionIT.class); + + private static final String TABLE_NAME = "anomaly_test"; + private static final String SINK_TABLE_NAME = "anomaly_results"; + + // Window size used in all metric specs (seconds). + private static final int WINDOW_SIZE_SEC = 1; + + // Baseline: insert 100 rows every second for ~6 minutes (360 batches). + // Each batch lands in one 1-second window, giving the detector many data + // points to warm up before the anomaly spike. + private static final int BASELINE_BATCHES = 360; + private static final int ROWS_PER_BATCH = 100; + private static final long BATCH_INTERVAL_MS = 1000; + private static final double NORMAL_AMOUNT = 10.0; + + // Anomaly: 100 rows with amount=10000.0 → window MEAN ≈ 10000.0 (1000x spike). + private static final double ANOMALY_AMOUNT = 10000.0; + + private BigQueryResourceManager bigQueryResourceManager; + private PubsubResourceManager pubsubResourceManager; + + @Before + public void setUp() throws IOException { + bigQueryResourceManager = + BigQueryResourceManager.builder(testName, PROJECT, credentials).build(); + pubsubResourceManager = + PubsubResourceManager.builder(testName, PROJECT, credentialsProvider).build(); + } + + @After + public void cleanUp() { + ResourceManagerUtils.cleanResources(bigQueryResourceManager, pubsubResourceManager); + } + + @Test + public void testDetectsAnomalyAndPublishesToPubSub() throws IOException, InterruptedException { + testSimpleSumMetric(); + } + + @Test + public void testGroupedRatioMetric() throws IOException, InterruptedException { + testGroupedRatioMetricImpl(/* mapperSidePrecombine= */ false); + } + + @Test + public void testGroupedRatioMetricWithPrecombine() throws IOException, InterruptedException { + testGroupedRatioMetricImpl(/* mapperSidePrecombine= */ true); + } + + @Test + public void testThresholdDetector() throws IOException, InterruptedException { + testThresholdDetectorImpl(); + } + + // ------------------------------------------------------------------------- + // Test implementations + // ------------------------------------------------------------------------- + + private void testSimpleSumMetric() throws IOException, InterruptedException { + // --- Arrange --- + + // Create BQ table. APPENDS() does not require change tracking. + Schema schema = + Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("amount", StandardSQLTypeName.FLOAT64)); + bigQueryResourceManager.createDataset(REGION); + bigQueryResourceManager.createTable(TABLE_NAME, schema); + + // Create Pub/Sub topic and subscription to verify anomaly output. + TopicName outputTopic = pubsubResourceManager.createTopic("anomaly-output"); + SubscriptionName outputSubscription = + pubsubResourceManager.createSubscription(outputTopic, "anomaly-output-sub"); + + // 1-second fixed windows, MEAN of amount, RobustZScore detector. + String metricSpec = + "{\"aggregation\":{\"window\":{\"type\":\"fixed\"," + + "\"size_seconds\":" + + WINDOW_SIZE_SEC + + "},\"measures\":[{\"field\":\"amount\"," + + "\"agg\":\"MEAN\",\"alias\":\"avg_amount\"}]}}"; + String detectorSpec = "{\"type\":\"RobustZScore\"}"; + + String tableRef = + String.format( + "%s:%s.%s", + bigQueryResourceManager.getProjectId(), + bigQueryResourceManager.getDatasetId(), + TABLE_NAME); + + String sinkTableRef = + String.format( + "%s:%s.%s", + bigQueryResourceManager.getProjectId(), + bigQueryResourceManager.getDatasetId(), + SINK_TABLE_NAME); + + // --- Act --- + + // Launch the pipeline first. + // start_offset_sec=300 ensures it reads data inserted before and after launch. + LaunchConfig.Builder options = + LaunchConfig.builder(testName, specPath) + .addParameter("table", tableRef) + .addParameter("metric_spec", metricSpec) + .addParameter("detector_spec", detectorSpec) + .addParameter("topic", outputTopic.toString()) + .addParameter("poll_interval_sec", "15") + .addParameter("start_offset_sec", "300") + .addParameter("duration_sec", "600") + .addParameter("log_all_results", "true") + .addParameter("sink_table", sinkTableRef); + + LaunchInfo info = launchTemplate(options); + assertThatPipeline(info).isRunning(); + + // Insert baseline data: 360 batches x 100 rows, one batch every second (~6 min). + // This gives the pipeline time to start workers and the detector time to warm up + // on many 1-second windows of normal data before the anomaly arrives. + LOG.info( + "Inserting {} batches of {} rows every {}ms (amount={})", + BASELINE_BATCHES, + ROWS_PER_BATCH, + BATCH_INTERVAL_MS, + NORMAL_AMOUNT); + Random rng = new Random(42); + int rowId = 0; + for (int batch = 0; batch < BASELINE_BATCHES; batch++) { + List rows = new ArrayList<>(); + for (int i = 0; i < ROWS_PER_BATCH; i++) { + // Tiny variance (stdev ~0.5) so MAD/stdev > 0 for the detector. + double amount = NORMAL_AMOUNT + rng.nextGaussian() * 0.5; + rows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", amount))); + } + bigQueryResourceManager.write(TABLE_NAME, rows); + if (batch < BASELINE_BATCHES - 1) { + TimeUnit.MILLISECONDS.sleep(BATCH_INTERVAL_MS); + } + if ((batch + 1) % 60 == 0) { + LOG.info("Inserted batch {}/{} ({} rows so far)", batch + 1, BASELINE_BATCHES, rowId); + } + } + LOG.info("Inserted {} baseline rows total", rowId); + + // Pause to ensure the anomaly lands in a clean window, not mixed with baseline. + TimeUnit.SECONDS.sleep(2); + + // Insert anomalous spike: same batch size but 1000x the amount. + LOG.info("Inserting anomalous batch ({} rows, amount={})", ROWS_PER_BATCH, ANOMALY_AMOUNT); + List anomalyRows = new ArrayList<>(); + for (int i = 0; i < ROWS_PER_BATCH; i++) { + anomalyRows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", ANOMALY_AMOUNT))); + } + bigQueryResourceManager.write(TABLE_NAME, anomalyRows); + LOG.info("Inserted {} anomalous rows", anomalyRows.size()); + + // --- Assert --- + + // Wait for at least 1 anomaly message on the Pub/Sub subscription. + PubsubMessagesCheck pubsubCheck = + PubsubMessagesCheck.builder(pubsubResourceManager, outputSubscription) + .setMinMessages(1) + .build(); + + Result result = pipelineOperator().waitForConditionAndCancel(createConfig(info), pubsubCheck); + assertThatResult(result).meetsConditions(); + + // Verify the anomaly message content. + List messages = pubsubCheck.getReceivedMessageList(); + assertThat(messages).isNotEmpty(); + + String messageData = messages.get(0).getMessage().getData().toStringUtf8(); + LOG.info("Received anomaly message: {}", messageData); + JSONObject payload = new JSONObject(messageData); + + assertThat(payload.getString("event_description")).contains("Anomaly detected"); + assertThat(payload.getString("agent_id")).isEqualTo("RobustZScore"); + + // --- Verify BQ sink table --- + verifySinkTable(SINK_TABLE_NAME, WINDOW_SIZE_SEC, null /* no keys expected */); + } + + /** + * Tests a grouped ratio metric (CTR = clicks / impressions) with anomaly detection. + * + *

Inserts baseline data with a stable click-through rate (~10%) for two campaign types, then + * injects an anomalous spike in one campaign's CTR (~90%), and verifies the pipeline detects the + * anomaly and publishes it to Pub/Sub with the correct key. + */ + private void testGroupedRatioMetricImpl(boolean mapperSidePrecombine) + throws IOException, InterruptedException { + // --- Arrange --- + + String tableName = "ctr_test"; + + Schema schema = + Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("campaign_type", StandardSQLTypeName.STRING), + Field.of("is_click", StandardSQLTypeName.INT64)); + bigQueryResourceManager.createDataset(REGION); + bigQueryResourceManager.createTable(tableName, schema); + + TopicName outputTopic = pubsubResourceManager.createTopic("ctr-anomaly-output"); + SubscriptionName outputSubscription = + pubsubResourceManager.createSubscription(outputTopic, "ctr-anomaly-output-sub"); + + String sinkTableName = "ctr_results"; + + // Grouped ratio: CTR = clicks / impressions per campaign_type. + String metricSpec = + "{\"aggregation\":{\"window\":{\"type\":\"fixed\"," + + "\"size_seconds\":" + + WINDOW_SIZE_SEC + + "},\"group_by\":[\"campaign_type\"]," + + "\"measures\":[{\"field\":\"is_click\",\"agg\":\"SUM\",\"alias\":\"clicks\"}," + + "{\"field\":\"is_click\",\"agg\":\"COUNT\",\"alias\":\"impressions\"}]}," + + "\"measure_combiner\":{\"expression\":\"clicks / impressions\"}}"; + String detectorSpec = "{\"type\":\"RobustZScore\"}"; + + String tableRef = + String.format( + "%s:%s.%s", + bigQueryResourceManager.getProjectId(), + bigQueryResourceManager.getDatasetId(), + tableName); + + String sinkTableRef = + String.format( + "%s:%s.%s", + bigQueryResourceManager.getProjectId(), + bigQueryResourceManager.getDatasetId(), + sinkTableName); + + // --- Act --- + + LaunchConfig.Builder options = + LaunchConfig.builder(testName, specPath) + .addParameter("table", tableRef) + .addParameter("metric_spec", metricSpec) + .addParameter("detector_spec", detectorSpec) + .addParameter("topic", outputTopic.toString()) + .addParameter("poll_interval_sec", "15") + .addParameter("start_offset_sec", "300") + .addParameter("duration_sec", "600") + .addParameter("log_all_results", "true") + .addParameter("sink_table", sinkTableRef); + + if (mapperSidePrecombine) { + options.addParameter("mapper_side_precombine", "true"); + } + + LaunchInfo info = launchTemplate(options); + assertThatPipeline(info).isRunning(); + + // Insert baseline data: CTR ~10% for both campaigns. + // Campaigns are round-robin (deterministic split), clicks are random + // to provide natural per-window variance needed by RobustZScore (MAD > 0). + // 500 rows per batch (250 per key) keeps per-window CTR tight (~0.08-0.12). + // 360 batches, one batch every second (~6 min). + int ctrRowsPerBatch = 500; + LOG.info( + "Inserting {} batches of {} rows every {}ms (CTR ~10%%)", + BASELINE_BATCHES, ctrRowsPerBatch, BATCH_INTERVAL_MS); + Random rng = new Random(42); + String[] campaigns = {"search", "display"}; + int rowId = 0; + for (int batch = 0; batch < BASELINE_BATCHES; batch++) { + List rows = new ArrayList<>(); + for (int i = 0; i < ctrRowsPerBatch; i++) { + String campaign = campaigns[i % campaigns.length]; + int isClick = rng.nextDouble() < 0.10 ? 1 : 0; + rows.add( + RowToInsert.of( + ImmutableMap.of("id", ++rowId, "campaign_type", campaign, "is_click", isClick))); + } + bigQueryResourceManager.write(tableName, rows); + if (batch < BASELINE_BATCHES - 1) { + TimeUnit.MILLISECONDS.sleep(BATCH_INTERVAL_MS); + } + if ((batch + 1) % 60 == 0) { + LOG.info("Inserted batch {}/{} ({} rows so far)", batch + 1, BASELINE_BATCHES, rowId); + } + } + LOG.info("Inserted {} baseline rows total", rowId); + + TimeUnit.SECONDS.sleep(2); + + // Inject anomaly: "search" campaign CTR spikes to ~90%. + LOG.info("Inserting anomalous batch ({} rows, search CTR ~90%%)", ctrRowsPerBatch); + List anomalyRows = new ArrayList<>(); + for (int i = 0; i < ctrRowsPerBatch; i++) { + int isClick = rng.nextDouble() < 0.90 ? 1 : 0; + anomalyRows.add( + RowToInsert.of( + ImmutableMap.of("id", ++rowId, "campaign_type", "search", "is_click", isClick))); + } + bigQueryResourceManager.write(tableName, anomalyRows); + LOG.info("Inserted {} anomalous rows", anomalyRows.size()); + + // --- Assert --- + + PubsubMessagesCheck pubsubCheck = + PubsubMessagesCheck.builder(pubsubResourceManager, outputSubscription) + .setMinMessages(1) + .build(); + + Result result = pipelineOperator().waitForConditionAndCancel(createConfig(info), pubsubCheck); + assertThatResult(result).meetsConditions(); + + List messages = pubsubCheck.getReceivedMessageList(); + assertThat(messages).isNotEmpty(); + + String messageData = messages.get(0).getMessage().getData().toStringUtf8(); + LOG.info("Received CTR anomaly message: {}", messageData); + JSONObject payload = new JSONObject(messageData); + + assertThat(payload.getString("event_description")).contains("Anomaly detected"); + assertThat(payload.getString("agent_id")).isEqualTo("RobustZScore"); + assertThat(payload.has("key")).isTrue(); + + // --- Verify BQ sink table --- + Set expectedKeys = Set.of("('search',)", "('display',)"); + verifySinkTable(sinkTableName, WINDOW_SIZE_SEC, expectedKeys); + } + + /** + * Tests the Threshold detector with a fixed expression (value >= 100). + * + *

Inserts rows with amount=10 (below threshold), then a batch with amount=500 (above + * threshold). No warmup period is needed — the threshold fires immediately on any value that + * satisfies the expression. + */ + private void testThresholdDetectorImpl() throws IOException, InterruptedException { + // --- Arrange --- + + String tableName = "threshold_test"; + + Schema schema = + Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("amount", StandardSQLTypeName.FLOAT64)); + bigQueryResourceManager.createDataset(REGION); + bigQueryResourceManager.createTable(tableName, schema); + + TopicName outputTopic = pubsubResourceManager.createTopic("threshold-output"); + SubscriptionName outputSubscription = + pubsubResourceManager.createSubscription(outputTopic, "threshold-output-sub"); + + String sinkTableName = "threshold_results"; + + // Simple MEAN metric with Threshold detector. + String metricSpec = + "{\"aggregation\":{\"window\":{\"type\":\"fixed\"," + + "\"size_seconds\":" + + WINDOW_SIZE_SEC + + "},\"measures\":[{\"field\":\"amount\"," + + "\"agg\":\"MEAN\",\"alias\":\"avg_amount\"}]}}"; + String detectorSpec = "{\"type\":\"Threshold\",\"expression\":\"value >= 100\"}"; + + String tableRef = + String.format( + "%s:%s.%s", + bigQueryResourceManager.getProjectId(), + bigQueryResourceManager.getDatasetId(), + tableName); + + String sinkTableRef = + String.format( + "%s:%s.%s", + bigQueryResourceManager.getProjectId(), + bigQueryResourceManager.getDatasetId(), + sinkTableName); + + // --- Act --- + + LaunchConfig.Builder options = + LaunchConfig.builder(testName, specPath) + .addParameter("table", tableRef) + .addParameter("metric_spec", metricSpec) + .addParameter("detector_spec", detectorSpec) + .addParameter("topic", outputTopic.toString()) + .addParameter("poll_interval_sec", "15") + .addParameter("start_offset_sec", "300") + .addParameter("duration_sec", "600") + .addParameter("log_all_results", "true") + .addParameter("sink_table", sinkTableRef); + + LaunchInfo info = launchTemplate(options); + assertThatPipeline(info).isRunning(); + + // Insert baseline data: amount=10, well below the threshold of 100. + // Fewer batches needed since no warmup — just enough for the pipeline to start. + int baselineBatches = 120; + LOG.info( + "Inserting {} batches of {} rows every {}ms (amount=10, below threshold)", + baselineBatches, + ROWS_PER_BATCH, + BATCH_INTERVAL_MS); + int rowId = 0; + for (int batch = 0; batch < baselineBatches; batch++) { + List rows = new ArrayList<>(); + for (int i = 0; i < ROWS_PER_BATCH; i++) { + rows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", 10.0))); + } + bigQueryResourceManager.write(tableName, rows); + if (batch < baselineBatches - 1) { + TimeUnit.MILLISECONDS.sleep(BATCH_INTERVAL_MS); + } + if ((batch + 1) % 60 == 0) { + LOG.info("Inserted batch {}/{} ({} rows so far)", batch + 1, baselineBatches, rowId); + } + } + LOG.info("Inserted {} baseline rows total", rowId); + + TimeUnit.SECONDS.sleep(2); + + // Insert above-threshold batch: amount=500, well above threshold of 100. + LOG.info("Inserting above-threshold batch ({} rows, amount=500)", ROWS_PER_BATCH); + List alertRows = new ArrayList<>(); + for (int i = 0; i < ROWS_PER_BATCH; i++) { + alertRows.add(RowToInsert.of(ImmutableMap.of("id", ++rowId, "amount", 500.0))); + } + bigQueryResourceManager.write(tableName, alertRows); + LOG.info("Inserted {} above-threshold rows", alertRows.size()); + + // --- Assert --- + + PubsubMessagesCheck pubsubCheck = + PubsubMessagesCheck.builder(pubsubResourceManager, outputSubscription) + .setMinMessages(1) + .build(); + + Result result = pipelineOperator().waitForConditionAndCancel(createConfig(info), pubsubCheck); + assertThatResult(result).meetsConditions(); + + List messages = pubsubCheck.getReceivedMessageList(); + assertThat(messages).isNotEmpty(); + + String messageData = messages.get(0).getMessage().getData().toStringUtf8(); + LOG.info("Received threshold alert message: {}", messageData); + JSONObject payload = new JSONObject(messageData); + + assertThat(payload.getString("event_description")).contains("Anomaly detected"); + assertThat(payload.getString("agent_id")).isEqualTo("Threshold(value >= 100)"); + + // --- Verify BQ sink table --- + verifySinkTable(sinkTableName, WINDOW_SIZE_SEC, null /* no keys expected */); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + /** + * Verifies the BQ sink table written by the pipeline. + * + * @param tableName sink table name + * @param windowSizeSec expected window duration in seconds + * @param expectedKeys if non-null, the set of expected key values; if null, key must be absent + */ + private void verifySinkTable(String tableName, int windowSizeSec, Set expectedKeys) { + TableResult tableResult = bigQueryResourceManager.readTable(tableName); + List> rows = BigQueryAsserts.tableResultToRecords(tableResult); + + assertThat(rows).isNotEmpty(); + LOG.info("Sink table '{}' has {} rows", tableName, rows.size()); + + boolean hasOutlier = false; + Set observedKeys = new HashSet<>(); + Set validLabels = Set.of(-2, 0, 1); + + for (Map row : rows) { + // All expected columns exist. + assertThat(row).containsKey("window_start"); + assertThat(row).containsKey("window_end"); + assertThat(row).containsKey("value"); + assertThat(row).containsKey("label"); + + // Window timestamps parse as valid ISO-8601 UTC. + String windowStart = row.get("window_start").toString(); + String windowEnd = row.get("window_end").toString(); + Instant start = Instant.parse(windowStart); + Instant end = Instant.parse(windowEnd); + + // Window duration matches expected size. + long durationSec = end.getEpochSecond() - start.getEpochSecond(); + assertThat(durationSec).isEqualTo(windowSizeSec); + + // Value is a valid number. + assertThat(row.get("value")).isNotNull(); + double value = ((Number) row.get("value")).doubleValue(); + assertThat(Double.isNaN(value)).isFalse(); + + // Label is valid. + int label = ((Number) row.get("label")).intValue(); + assertThat(validLabels).contains(label); + + if (label == 1) { + hasOutlier = true; + } + + // Track keys. + if (row.containsKey("key") && row.get("key") != null) { + observedKeys.add(row.get("key").toString()); + } + } + + // At least one outlier was detected. + assertThat(hasOutlier).isTrue(); + + // Verify keys. + if (expectedKeys != null) { + assertThat(observedKeys).isEqualTo(expectedKeys); + } else { + assertThat(observedKeys).isEmpty(); + } + + LOG.info( + "Sink table '{}' verification passed ({} rows, outlier found)", tableName, rows.size()); + } +} diff --git a/python/src/test/python/bigquery-anomaly-detection/__init__.py b/python/src/test/python/bigquery-anomaly-detection/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/src/test/python/bigquery-anomaly-detection/metric_test.py b/python/src/test/python/bigquery-anomaly-detection/metric_test.py new file mode 100644 index 0000000000..30efeae123 --- /dev/null +++ b/python/src/test/python/bigquery-anomaly-detection/metric_test.py @@ -0,0 +1,524 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Unit tests for bqmonitor.metric.""" + +import json +import logging +import unittest + +logging.basicConfig(level=logging.INFO) + +import apache_beam as beam +from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to +from apache_beam.transforms import window as beam_window +from apache_beam.utils.windowed_value import WindowedValue +from apache_beam.utils.timestamp import Timestamp + +from bqmonitor.metric import AggOp +from bqmonitor.metric import AggregationSpec +from bqmonitor.metric import ComputeMetric +from bqmonitor.metric import DerivedField +from bqmonitor.metric import FanoutStrategy +from bqmonitor.metric import MeasureSpec +from bqmonitor.metric import MetricSpec +from bqmonitor.metric import WindowSpec +from bqmonitor.metric import WindowType +from bqmonitor.metric import _MapperSidePrecombine +from bqmonitor.metric import _PostCombineFn +from bqmonitor.metric import _SumCombineFn +from bqmonitor.safe_eval import Expr + + +class MetricSpecValidationTest(unittest.TestCase): + """Tests for MetricSpec validation.""" + + def _simple_spec(self, **kwargs): + defaults = dict( + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=60), + measures=[MeasureSpec(field='amount', agg=AggOp.SUM, + alias='total')], + )) + defaults.update(kwargs) + return MetricSpec(**defaults) + + def test_simple_spec_valid(self): + spec = self._simple_spec() + self.assertEqual(len(spec.aggregation.measures), 1) + + def test_no_measures_raises(self): + # @specifiable uses lazy init; access an attribute to trigger validation. + with self.assertRaises(ValueError) as ctx: + spec = MetricSpec(aggregation=AggregationSpec(measures=[])) + _ = spec.aggregation + self.assertIn('at least one measure', str(ctx.exception)) + + def test_multiple_measures_without_combiner_raises(self): + with self.assertRaises(ValueError) as ctx: + spec = MetricSpec(aggregation=AggregationSpec( + measures=[ + MeasureSpec(field='a', agg=AggOp.SUM, alias='x'), + MeasureSpec(field='b', agg=AggOp.COUNT, alias='y'), + ])) + _ = spec.aggregation + self.assertIn('measure_combiner is required', str(ctx.exception)) + + def test_multiple_measures_with_combiner(self): + spec = MetricSpec( + aggregation=AggregationSpec( + measures=[ + MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), + MeasureSpec(field='a', agg=AggOp.COUNT, alias='impressions'), + ]), + measure_combiner=Expr('clicks / impressions')) + self.assertIsNotNone(spec.measure_combiner) + + def test_combiner_unknown_field_raises(self): + with self.assertRaises(ValueError) as ctx: + spec = MetricSpec( + aggregation=AggregationSpec( + measures=[ + MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), + ]), + measure_combiner=Expr('clicks / impressions')) + _ = spec.aggregation + self.assertIn('unknown fields', str(ctx.exception)) + + def test_sliding_without_period_raises(self): + with self.assertRaises(ValueError) as ctx: + spec = MetricSpec(aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.SLIDING, size_seconds=60), + measures=[MeasureSpec(field='a', agg=AggOp.SUM, alias='x')])) + _ = spec.aggregation + self.assertIn('period_seconds', str(ctx.exception)) + + def test_sliding_with_period(self): + spec = MetricSpec(aggregation=AggregationSpec( + window=WindowSpec( + type=WindowType.SLIDING, size_seconds=60, period_seconds=10), + measures=[MeasureSpec(field='a', agg=AggOp.SUM, alias='x')])) + self.assertEqual(spec.aggregation.window.period_seconds, 10) + + +class MetricSpecRequiredColumnsTest(unittest.TestCase): + """Tests for required_source_columns().""" + + def test_simple_sum(self): + spec = MetricSpec(aggregation=AggregationSpec( + measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) + self.assertEqual(spec.required_source_columns(), {'amount'}) + + def test_count_excludes_field(self): + spec = MetricSpec(aggregation=AggregationSpec( + measures=[MeasureSpec(field='x', agg=AggOp.COUNT, alias='cnt')])) + self.assertEqual(spec.required_source_columns(), set()) + + def test_group_by_included(self): + spec = MetricSpec(aggregation=AggregationSpec( + group_by=['region', 'product'], + measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) + self.assertEqual( + spec.required_source_columns(), {'region', 'product', 'amount'}) + + def test_derived_field_references(self): + spec = MetricSpec( + aggregation=AggregationSpec( + measures=[MeasureSpec( + field='is_success', agg=AggOp.SUM, alias='successes')]), + derived_fields=[ + DerivedField( + name='is_success', + expression=Expr("1 if status == 'ok' else 0")) + ]) + # 'is_success' is derived, so excluded; 'status' is a source ref. + self.assertEqual(spec.required_source_columns(), {'status'}) + + def test_ctr_metric(self): + spec = MetricSpec( + aggregation=AggregationSpec( + group_by=['campaign_type'], + measures=[ + MeasureSpec(field='is_click', agg=AggOp.SUM, alias='clicks'), + MeasureSpec( + field='is_click', agg=AggOp.COUNT, alias='impressions'), + ]), + measure_combiner=Expr('clicks / impressions')) + # is_click (from SUM), campaign_type (from group_by). + # COUNT's field is excluded. + self.assertEqual( + spec.required_source_columns(), {'is_click', 'campaign_type'}) + + +class MetricSpecFromDictTest(unittest.TestCase): + """Tests for MetricSpec.from_dict() deserialization.""" + + def test_simple(self): + d = { + 'aggregation': { + 'window': {'type': 'fixed', 'size_seconds': 60}, + 'measures': [ + {'field': 'amount', 'agg': 'SUM', 'alias': 'total'}], + }} + spec = MetricSpec.from_dict(d) + self.assertEqual(spec.aggregation.window.type, WindowType.FIXED) + self.assertEqual(spec.aggregation.window.size_seconds, 60) + self.assertEqual(len(spec.aggregation.measures), 1) + self.assertEqual(spec.aggregation.measures[0].agg, AggOp.SUM) + + def test_with_combiner(self): + d = { + 'aggregation': { + 'measures': [ + {'field': 'x', 'agg': 'SUM', 'alias': 'clicks'}, + {'field': 'x', 'agg': 'COUNT', 'alias': 'impressions'}], + }, + 'measure_combiner': {'expression': 'clicks / impressions'}, + } + spec = MetricSpec.from_dict(d) + self.assertIsNotNone(spec.measure_combiner) + self.assertEqual(spec.measure_combiner.field_refs(), {'clicks', 'impressions'}) + + def test_with_derived_fields(self): + d = { + 'aggregation': { + 'measures': [ + {'field': 'is_ok', 'agg': 'SUM', 'alias': 'ok_count'}], + }, + 'derived_fields': [ + {'name': 'is_ok', 'expression': "1 if status == 'ok' else 0"}], + } + spec = MetricSpec.from_dict(d) + self.assertEqual(len(spec.derived_fields), 1) + self.assertEqual(spec.derived_fields[0].name, 'is_ok') + + def test_roundtrip_json(self): + """from_dict(to_dict(spec)) should produce an equivalent spec.""" + original = MetricSpec( + aggregation=AggregationSpec( + window=WindowSpec( + type=WindowType.SLIDING, size_seconds=300, period_seconds=60), + group_by=['region'], + measures=[ + MeasureSpec(field='a', agg=AggOp.SUM, alias='clicks'), + MeasureSpec(field='a', agg=AggOp.COUNT, alias='impressions'), + ]), + measure_combiner=Expr('clicks / impressions'), + name='ctr') + d = original.to_dict() + # Verify JSON-serializable. + json_str = json.dumps(d) + restored = MetricSpec.from_dict(json.loads(json_str)) + self.assertEqual(restored.name, 'ctr') + self.assertEqual(restored.aggregation.window.type, WindowType.SLIDING) + self.assertEqual(restored.aggregation.window.period_seconds, 60) + self.assertEqual(len(restored.aggregation.measures), 2) + self.assertEqual( + restored.required_source_columns(), {'a', 'region'}) + + +class MetricSpecToDictTest(unittest.TestCase): + """Tests for MetricSpec.to_dict() serialization.""" + + def test_simple(self): + spec = MetricSpec(aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=60), + measures=[MeasureSpec(field='amount', agg=AggOp.SUM, alias='total')])) + d = spec.to_dict() + self.assertEqual(d['aggregation']['window']['type'], 'fixed') + self.assertEqual(d['aggregation']['measures'][0]['agg'], 'SUM') + + def test_excludes_optional_none(self): + spec = MetricSpec(aggregation=AggregationSpec( + measures=[MeasureSpec(field='x', agg=AggOp.SUM, alias='y')])) + d = spec.to_dict() + self.assertNotIn('derived_fields', d) + self.assertNotIn('measure_combiner', d) + self.assertNotIn('name', d) + + +class MapperSidePrecombineDoFnTest(unittest.TestCase): + """Unit tests for _MapperSidePrecombine DoFn.""" + + def _run_precombine(self, combine_fn, windowed_values, max_keys=100_000): + """Run _MapperSidePrecombine on a list of WindowedValues, return results.""" + dofn = _MapperSidePrecombine(combine_fn, max_keys=max_keys) + dofn.setup() + dofn.start_bundle() + mid_results = [] + for wv in windowed_values: + result = dofn.process( + wv.value, window=wv.windows[0], timestamp=wv.timestamp) + if result: + mid_results.extend(result) + finish_results = list(dofn.finish_bundle()) + dofn.teardown() + return mid_results + finish_results + + def _make_wv(self, key, value, timestamp=0, window=None): + if window is None: + window = beam_window.GlobalWindow() + return WindowedValue((key, value), Timestamp(timestamp), (window,)) + + def test_single_key_sums(self): + """Multiple values for one key should be folded into one accumulator.""" + elements = [ + self._make_wv('a', 10), + self._make_wv('a', 20), + self._make_wv('a', 30), + ] + results = self._run_precombine(_SumCombineFn(), elements) + self.assertEqual(len(results), 1) + key, acc = results[0].value + self.assertEqual(key, 'a') + self.assertEqual(acc, 60) + + def test_multiple_keys(self): + """Each key gets its own accumulator.""" + elements = [ + self._make_wv('a', 1), + self._make_wv('b', 2), + self._make_wv('a', 3), + self._make_wv('b', 4), + ] + results = self._run_precombine(_SumCombineFn(), elements) + result_dict = {wv.value[0]: wv.value[1] for wv in results} + self.assertEqual(result_dict, {'a': 4, 'b': 6}) + + def test_fixed_windows_separate(self): + """Same key in different windows should produce separate accumulators.""" + w1 = beam_window.IntervalWindow(0, 60) + w2 = beam_window.IntervalWindow(60, 120) + elements = [ + self._make_wv('a', 10, timestamp=5, window=w1), + self._make_wv('a', 20, timestamp=70, window=w2), + self._make_wv('a', 30, timestamp=15, window=w1), + ] + results = self._run_precombine(_SumCombineFn(), elements) + self.assertEqual(len(results), 2) + by_window = {wv.windows[0]: wv.value for wv in results} + self.assertEqual(by_window[w1], ('a', 40)) + self.assertEqual(by_window[w2], ('a', 20)) + + def test_sliding_windows_separate(self): + """An element assigned to overlapping sliding windows produces separate + accumulators per window (upstream WindowInto does the expansion).""" + w1 = beam_window.IntervalWindow(0, 60) + w2 = beam_window.IntervalWindow(10, 70) + # Simulate WindowInto expansion: one element in two windows. + elements = [ + self._make_wv('a', 5, timestamp=15, window=w1), + self._make_wv('a', 5, timestamp=15, window=w2), + self._make_wv('a', 3, timestamp=25, window=w1), + self._make_wv('a', 3, timestamp=25, window=w2), + ] + results = self._run_precombine(_SumCombineFn(), elements) + self.assertEqual(len(results), 2) + by_window = {wv.windows[0]: wv.value[1] for wv in results} + self.assertEqual(by_window[w1], 8) + self.assertEqual(by_window[w2], 8) + + def test_timestamp_preserves_earliest(self): + """Output timestamp should be the earliest of all folded elements.""" + w = beam_window.IntervalWindow(0, 60) + elements = [ + self._make_wv('a', 1, timestamp=30, window=w), + self._make_wv('a', 2, timestamp=10, window=w), + self._make_wv('a', 3, timestamp=50, window=w), + ] + results = self._run_precombine(_SumCombineFn(), elements) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].timestamp, Timestamp(10)) + + def test_window_metadata_preserved(self): + """Output WindowedValue should carry the correct window.""" + w = beam_window.IntervalWindow(100, 200) + elements = [self._make_wv('k', 42, timestamp=150, window=w)] + results = self._run_precombine(_SumCombineFn(), elements) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].windows, (w,)) + + def test_eviction_under_pressure(self): + """When max_keys is hit, entries are evicted (yielded from process).""" + # max_keys=3, insert 4 distinct keys → should evict some during process. + elements = [ + self._make_wv('a', 1), + self._make_wv('b', 2), + self._make_wv('c', 3), + self._make_wv('d', 4), + ] + results = self._run_precombine(_SumCombineFn(), elements, max_keys=3) + result_dict = {wv.value[0]: wv.value[1] for wv in results} + self.assertEqual(result_dict, {'a': 1, 'b': 2, 'c': 3, 'd': 4}) + + def test_evicted_values_not_lost(self): + """Values added to evicted keys before eviction are in the output.""" + elements = [ + self._make_wv('a', 10), + self._make_wv('a', 20), # a=30 now + self._make_wv('b', 5), + self._make_wv('c', 7), + # max_keys=2, so inserting 'c' triggers eviction of 'a' or 'b'. + # After eviction, 'a' or 'b' is yielded mid-process. + ] + results = self._run_precombine(_SumCombineFn(), elements, max_keys=2) + result_dict = {wv.value[0]: wv.value[1] for wv in results} + self.assertEqual(result_dict['a'], 30) + self.assertEqual(result_dict['b'], 5) + self.assertEqual(result_dict['c'], 7) + + def test_empty_bundle(self): + """No elements → no output.""" + results = self._run_precombine(_SumCombineFn(), []) + self.assertEqual(results, []) + + +class MapperSidePrecombinePipelineTest(unittest.TestCase): + """Integration tests: _MapperSidePrecombine + _PostCombineFn in a pipeline.""" + + _STREAMING_OPTIONS = beam.options.pipeline_options.PipelineOptions( + ['--streaming']) + + def test_keyed_sum_with_precombine(self): + """Precombine + CombinePerKey(_PostCombineFn) produces correct sums.""" + spec = MetricSpec( + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=60), + group_by=['region'], + measures=[MeasureSpec(field='amount', agg=AggOp.SUM, + alias='total')])) + rows = [ + {'region': 'us', 'amount': 10}, + {'region': 'us', 'amount': 20}, + {'region': 'eu', 'amount': 5}, + {'region': 'eu', 'amount': 15}, + {'region': 'us', 'amount': 30}, + ] + with TestPipeline(options=self._STREAMING_OPTIONS) as p: + timestamped = ( + p + | beam.Create(rows) + | beam.Map(lambda r: beam_window.TimestampedValue(r, 10))) + result = timestamped | ComputeMetric(spec, + mapper_side_precombine=True) + totals = result | beam.MapTuple( + lambda k, row: (k, row.value)) + + assert_that(totals, equal_to([ + (('us',), 60.0), + (('eu',), 20.0), + ])) + + def test_global_sum_with_precombine(self): + """Precombine + CombineGlobally(_PostCombineFn) produces correct sum.""" + spec = MetricSpec( + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=60), + measures=[MeasureSpec(field='amount', agg=AggOp.SUM, + alias='total')])) + rows = [{'amount': v} for v in [10, 20, 30, 40]] + with TestPipeline(options=self._STREAMING_OPTIONS) as p: + timestamped = ( + p + | beam.Create(rows) + | beam.Map(lambda r: beam_window.TimestampedValue(r, 10))) + result = timestamped | ComputeMetric( + spec, + fanout_strategy=FanoutStrategy.NONE, + mapper_side_precombine=True) + totals = result | beam.Map(lambda row: row.value) + assert_that(totals, equal_to([100.0])) + + def test_multi_measure_with_precombine(self): + """Precombine works with TupleCombineFn (multiple measures).""" + spec = MetricSpec( + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=60), + group_by=['region'], + measures=[ + MeasureSpec(field='amount', agg=AggOp.SUM, alias='total'), + MeasureSpec(field='amount', agg=AggOp.COUNT, alias='cnt'), + ]), + measure_combiner=Expr('total / cnt')) + rows = [ + {'region': 'us', 'amount': 10}, + {'region': 'us', 'amount': 30}, + ] + with TestPipeline(options=self._STREAMING_OPTIONS) as p: + timestamped = ( + p + | beam.Create(rows) + | beam.Map(lambda r: beam_window.TimestampedValue(r, 10))) + result = timestamped | ComputeMetric(spec, + mapper_side_precombine=True) + values = result | beam.MapTuple(lambda k, row: row.value) + assert_that(values, equal_to([20.0])) + + def test_sliding_window_with_precombine(self): + """Precombine correctly separates accumulators across overlapping windows.""" + spec = MetricSpec( + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.SLIDING, + size_seconds=30, period_seconds=10), + group_by=['key'], + measures=[MeasureSpec(field='v', agg=AggOp.SUM, alias='total')])) + rows = [ + {'key': 'a', 'v': 10}, + {'key': 'a', 'v': 20}, + ] + # t=105 with size=30, period=10 → windows [80,110), [90,120), [100,130) + with TestPipeline(options=beam.options.pipeline_options.PipelineOptions( + ['--streaming'])) as p: + timestamped = ( + p + | beam.Create(rows) + | beam.Map(lambda r: beam_window.TimestampedValue(r, 105))) + result = timestamped | ComputeMetric(spec, + mapper_side_precombine=True) + totals = result | beam.MapTuple(lambda _, row: row.value) + # Each of the 3 sliding windows sees both elements → sum=30. + assert_that(totals, equal_to([30.0, 30.0, 30.0])) + + def test_precombine_matches_no_precombine(self): + """Precombine=True produces same results as precombine=False.""" + spec = MetricSpec( + aggregation=AggregationSpec( + window=WindowSpec(type=WindowType.FIXED, size_seconds=60), + group_by=['key'], + measures=[MeasureSpec(field='v', agg=AggOp.SUM, alias='total')])) + rows = [{'key': k, 'v': v} + for k in ['a', 'b', 'c'] + for v in range(1, 11)] + + def run_pipeline(precombine): + with TestPipeline(options=self._STREAMING_OPTIONS) as p: + timestamped = ( + p + | beam.Create(rows) + | beam.Map(lambda r: beam_window.TimestampedValue(r, 10))) + result = timestamped | ComputeMetric(spec, + mapper_side_precombine=precombine) + totals = result | beam.MapTuple(lambda _, row: row.value) + assert_that(totals, equal_to([55.0, 55.0, 55.0])) + + run_pipeline(False) + run_pipeline(True) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py b/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py new file mode 100644 index 0000000000..3f654893d4 --- /dev/null +++ b/python/src/test/python/bigquery-anomaly-detection/pipeline_test.py @@ -0,0 +1,287 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Unit tests for bqmonitor.pipeline helpers.""" + +import json +import logging +import unittest + +logging.basicConfig(level=logging.INFO) + +import apache_beam as beam +from apache_beam.ml.anomaly.base import AnomalyPrediction +from apache_beam.ml.anomaly.base import AnomalyResult + +from bqmonitor.pipeline import _FormatAnomalyAsJson +from bqmonitor.pipeline import _FormatResultForBQ +from bqmonitor.pipeline import _parse_detector_spec +from bqmonitor.pipeline import _parse_table_ref +from bqmonitor.pipeline import _ThresholdAlert +from bqmonitor.pipeline import _unpack_result + + +class ParseTableRefTest(unittest.TestCase): + """Tests for _parse_table_ref().""" + + def test_colon_format(self): + p, d, t = _parse_table_ref('my-project:my_dataset.my_table') + self.assertEqual(p, 'my-project') + self.assertEqual(d, 'my_dataset') + self.assertEqual(t, 'my_table') + + def test_dot_format(self): + p, d, t = _parse_table_ref('my-project.my_dataset.my_table') + self.assertEqual(p, 'my-project') + self.assertEqual(d, 'my_dataset') + self.assertEqual(t, 'my_table') + + def test_invalid_format_raises(self): + with self.assertRaises(ValueError): + _parse_table_ref('not_valid') + + def test_empty_raises(self): + with self.assertRaises(ValueError): + _parse_table_ref('') + + +class UnpackResultTest(unittest.TestCase): + """Tests for _unpack_result().""" + + def test_keyed(self): + result = object() + key, r = _unpack_result(('mykey', result)) + self.assertEqual(key, 'mykey') + self.assertIs(r, result) + + def test_unkeyed(self): + result = object() + key, r = _unpack_result(result) + self.assertIsNone(key) + self.assertIs(r, result) + + +class ParseDetectorSpecTest(unittest.TestCase): + """Tests for _parse_detector_spec().""" + + def test_zscore(self): + detector = _parse_detector_spec('{"type":"ZScore"}') + self.assertEqual(type(detector).__name__, 'ZScore') + + def test_iqr(self): + detector = _parse_detector_spec('{"type":"IQR"}') + self.assertEqual(type(detector).__name__, 'IQR') + + def test_robust_zscore(self): + detector = _parse_detector_spec('{"type":"RobustZScore"}') + self.assertEqual(type(detector).__name__, 'RobustZScore') + + def test_threshold(self): + detector = _parse_detector_spec( + '{"type":"Threshold","expression":"value >= 100"}') + self.assertIsInstance(detector, _ThresholdAlert) + + def test_threshold_missing_expression_raises(self): + with self.assertRaises(ValueError) as ctx: + _parse_detector_spec('{"type":"Threshold"}') + self.assertIn('expression', str(ctx.exception)) + + def test_threshold_invalid_expression_raises(self): + with self.assertRaises(ValueError) as ctx: + _parse_detector_spec( + '{"type":"Threshold","expression":"import os"}') + self.assertIn('Invalid threshold expression', str(ctx.exception)) + + def test_unknown_type_raises(self): + with self.assertRaises(ValueError) as ctx: + _parse_detector_spec('{"type":"Unknown"}') + self.assertIn('Unknown', str(ctx.exception)) + + def test_invalid_json_raises(self): + with self.assertRaises(ValueError) as ctx: + _parse_detector_spec('{bad json}') + self.assertIn('Invalid JSON', str(ctx.exception)) + + def test_missing_type_raises(self): + with self.assertRaises(ValueError): + _parse_detector_spec('{"config":{}}') + + def test_zscore_with_threshold(self): + spec = json.dumps({ + 'type': 'ZScore', + 'config': { + 'threshold_criterion': { + 'type': 'FixedThreshold', + 'config': {'cutoff': 10}}}}) + detector = _parse_detector_spec(spec) + self.assertEqual(type(detector).__name__, 'ZScore') + + +class ThresholdAlertTest(unittest.TestCase): + """Tests for _ThresholdAlert DoFn.""" + + def _make_row(self, value): + return beam.Row(value=value, window_start=0.0, window_end=1.0) + + def _run_dofn(self, expression, element): + dofn = _ThresholdAlert(expression) + dofn.setup() + return list(dofn.process(element)) + + def test_above_threshold(self): + results = self._run_dofn('value >= 100', self._make_row(500.0)) + self.assertEqual(len(results), 1) + result = results[0] + self.assertIsInstance(result, AnomalyResult) + self.assertEqual(result.predictions[0].label, 1) + self.assertIsNone(result.predictions[0].score) + self.assertEqual( + result.predictions[0].model_id, 'Threshold(value >= 100)') + + def test_below_threshold(self): + results = self._run_dofn('value >= 100', self._make_row(50.0)) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].predictions[0].label, 0) + + def test_keyed_element(self): + row = self._make_row(200.0) + results = self._run_dofn('value >= 100', ('mykey', row)) + self.assertEqual(len(results), 1) + key, result = results[0] + self.assertEqual(key, 'mykey') + self.assertEqual(result.predictions[0].label, 1) + + def test_range_expression(self): + dofn = _ThresholdAlert('value > 100 or value < -100') + dofn.setup() + + # Above range. + results = list(dofn.process(self._make_row(200.0))) + self.assertEqual(results[0].predictions[0].label, 1) + + # Below range. + results = list(dofn.process(self._make_row(-200.0))) + self.assertEqual(results[0].predictions[0].label, 1) + + # Within range. + results = list(dofn.process(self._make_row(50.0))) + self.assertEqual(results[0].predictions[0].label, 0) + + def test_less_than_threshold(self): + results = self._run_dofn('value <= 0.01', self._make_row(0.005)) + self.assertEqual(results[0].predictions[0].label, 1) + + results = self._run_dofn('value <= 0.01', self._make_row(0.5)) + self.assertEqual(results[0].predictions[0].label, 0) + + +class FormatAnomalyAsJsonTest(unittest.TestCase): + """Tests for _FormatAnomalyAsJson DoFn.""" + + def _make_result(self, label, value=42.0, score=5.0, model_id='TestModel'): + row = beam.Row(value=value, window_start=1000.0, window_end=1001.0) + prediction = AnomalyPrediction( + model_id=model_id, score=score, label=label) + return AnomalyResult(example=row, predictions=[prediction]) + + def test_outlier_emits_json(self): + dofn = _FormatAnomalyAsJson() + results = list(dofn.process(self._make_result(label=1))) + self.assertEqual(len(results), 1) + payload = json.loads(results[0]) + self.assertIn('Anomaly detected', payload['event_description']) + self.assertEqual(payload['agent_id'], 'TestModel') + + def test_normal_emits_nothing(self): + dofn = _FormatAnomalyAsJson() + results = list(dofn.process(self._make_result(label=0))) + self.assertEqual(len(results), 0) + + def test_warmup_emits_nothing(self): + dofn = _FormatAnomalyAsJson() + results = list(dofn.process(self._make_result(label=-2))) + self.assertEqual(len(results), 0) + + def test_keyed_outlier_includes_key(self): + dofn = _FormatAnomalyAsJson() + result = self._make_result(label=1) + outputs = list(dofn.process(('campaign_search', result))) + self.assertEqual(len(outputs), 1) + payload = json.loads(outputs[0]) + self.assertEqual(payload['key'], 'campaign_search') + + def test_threshold_model_id(self): + dofn = _FormatAnomalyAsJson() + result = self._make_result( + label=1, model_id='Threshold(value >= 100)') + outputs = list(dofn.process(result)) + payload = json.loads(outputs[0]) + self.assertEqual(payload['agent_id'], 'Threshold(value >= 100)') + + +class FormatResultForBQTest(unittest.TestCase): + """Tests for _FormatResultForBQ DoFn.""" + + def _make_result(self, label, value=42.0, score=5.0): + row = beam.Row(value=value, window_start=1000.0, window_end=1001.0) + prediction = AnomalyPrediction( + model_id='TestModel', score=score, label=label) + return AnomalyResult(example=row, predictions=[prediction]) + + def test_outlier_row(self): + dofn = _FormatResultForBQ() + results = list(dofn.process(self._make_result(label=1, value=99.0, + score=4.5))) + self.assertEqual(len(results), 1) + row = results[0] + self.assertAlmostEqual(row['value'], 99.0) + self.assertAlmostEqual(row['score'], 4.5) + self.assertEqual(row['label'], 1) + self.assertIn('window_start', row) + self.assertIn('window_end', row) + self.assertNotIn('key', row) + + def test_normal_row(self): + dofn = _FormatResultForBQ() + results = list(dofn.process(self._make_result(label=0))) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['label'], 0) + + def test_warmup_row(self): + dofn = _FormatResultForBQ() + results = list(dofn.process(self._make_result(label=-2))) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['label'], -2) + + def test_keyed_row_includes_key(self): + dofn = _FormatResultForBQ() + result = self._make_result(label=1) + outputs = list(dofn.process(('campaign_search', result))) + self.assertEqual(len(outputs), 1) + self.assertEqual(outputs[0]['key'], 'campaign_search') + + def test_none_score(self): + row = beam.Row(value=10.0, window_start=0.0, window_end=1.0) + prediction = AnomalyPrediction( + model_id='Test', score=None, label=0) + result = AnomalyResult(example=row, predictions=[prediction]) + dofn = _FormatResultForBQ() + outputs = list(dofn.process(result)) + self.assertIsNone(outputs[0]['score']) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt b/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt new file mode 100644 index 0000000000..57726014e9 --- /dev/null +++ b/python/src/test/python/bigquery-anomaly-detection/requirements-test.txt @@ -0,0 +1 @@ +apache-beam[gcp,test] diff --git a/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py b/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py new file mode 100644 index 0000000000..974b10511f --- /dev/null +++ b/python/src/test/python/bigquery-anomaly-detection/safe_eval_test.py @@ -0,0 +1,244 @@ +# +# Copyright (C) 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +"""Unit tests for bqmonitor.safe_eval.""" + +import logging +import unittest + +logging.basicConfig(level=logging.INFO) + +from bqmonitor.safe_eval import Expr + + +class ExprArithmeticTest(unittest.TestCase): + """Tests for arithmetic operations.""" + + def test_addition(self): + self.assertEqual(Expr('a + b')({'a': 3, 'b': 4}), 7) + + def test_subtraction(self): + self.assertEqual(Expr('a - b')({'a': 10, 'b': 3}), 7) + + def test_multiplication(self): + self.assertEqual(Expr('a * b')({'a': 5, 'b': 6}), 30) + + def test_division(self): + self.assertAlmostEqual(Expr('a / b')({'a': 10, 'b': 3}), 10 / 3) + + def test_floor_division(self): + self.assertEqual(Expr('a // b')({'a': 10, 'b': 3}), 3) + + def test_modulo(self): + self.assertEqual(Expr('a % b')({'a': 10, 'b': 3}), 1) + + def test_power(self): + self.assertEqual(Expr('x ** 2')({'x': 5}), 25) + + def test_negation(self): + self.assertEqual(Expr('-x')({'x': 7}), -7) + + def test_parentheses(self): + self.assertEqual(Expr('(a + b) * c')({'a': 2, 'b': 3, 'c': 4}), 20) + + +class ExprComparisonTest(unittest.TestCase): + """Tests for comparison operations.""" + + def test_eq(self): + self.assertTrue(Expr('x == 1')({'x': 1})) + self.assertFalse(Expr('x == 1')({'x': 2})) + + def test_neq(self): + self.assertTrue(Expr('x != 1')({'x': 2})) + + def test_lt(self): + self.assertTrue(Expr('x < 5')({'x': 3})) + self.assertFalse(Expr('x < 5')({'x': 5})) + + def test_lte(self): + self.assertTrue(Expr('x <= 5')({'x': 5})) + + def test_gt(self): + self.assertTrue(Expr('x > 5')({'x': 6})) + + def test_gte(self): + self.assertTrue(Expr('x >= 5')({'x': 5})) + + def test_string_comparison(self): + self.assertTrue(Expr("s == 'ok'")({'s': 'ok'})) + self.assertFalse(Expr("s == 'ok'")({'s': 'fail'})) + + +class ExprBooleanTest(unittest.TestCase): + """Tests for boolean logic.""" + + def test_and(self): + self.assertTrue(Expr('a > 0 and b > 0')({'a': 1, 'b': 1})) + self.assertFalse(Expr('a > 0 and b > 0')({'a': 1, 'b': -1})) + + def test_or(self): + self.assertTrue(Expr('a > 0 or b > 0')({'a': -1, 'b': 1})) + self.assertFalse(Expr('a > 0 or b > 0')({'a': -1, 'b': -1})) + + def test_not(self): + self.assertTrue(Expr('not failed')({'failed': False})) + self.assertFalse(Expr('not failed')({'failed': True})) + + def test_compound(self): + e = Expr("x > 0 and not disabled or override == 'yes'") + self.assertTrue(e({'x': 1, 'disabled': False, 'override': 'no'})) + self.assertTrue(e({'x': -1, 'disabled': True, 'override': 'yes'})) + self.assertFalse(e({'x': -1, 'disabled': True, 'override': 'no'})) + + +class ExprIfElseTest(unittest.TestCase): + """Tests for conditional expressions.""" + + def test_if_else(self): + e = Expr("1 if status == 'ok' else 0") + self.assertEqual(e({'status': 'ok'}), 1) + self.assertEqual(e({'status': 'fail'}), 0) + + def test_if_else_with_bool(self): + e = Expr("1 if a > 0 and b > 0 else 0") + self.assertEqual(e({'a': 1, 'b': 1}), 1) + self.assertEqual(e({'a': 1, 'b': -1}), 0) + + +class ExprBuiltinsTest(unittest.TestCase): + """Tests for safe builtin functions.""" + + def test_abs(self): + self.assertEqual(Expr('abs(x)')({'x': -7}), 7) + self.assertEqual(Expr('abs(x)')({'x': 7}), 7) + + def test_min(self): + self.assertEqual(Expr('min(a, b)')({'a': 3, 'b': 5}), 3) + + def test_max(self): + self.assertEqual(Expr('max(a, b)')({'a': 3, 'b': 5}), 5) + + def test_max_with_literal(self): + self.assertEqual(Expr('max(x, 1)')({'x': 0}), 1) + + def test_round(self): + self.assertAlmostEqual(Expr('round(x, 2)')({'x': 3.14159}), 3.14) + + def test_nested_builtins(self): + self.assertEqual(Expr('max(abs(a), abs(b))')({'a': -5, 'b': 3}), 5) + + +class ExprFieldRefsTest(unittest.TestCase): + """Tests for field_refs() extraction.""" + + def test_simple(self): + self.assertEqual(Expr('a + b').field_refs(), {'a', 'b'}) + + def test_excludes_builtins(self): + self.assertEqual( + Expr('max(clicks, 1) / abs(total)').field_refs(), + {'clicks', 'total'}) + + def test_if_else_refs(self): + self.assertEqual( + Expr("1 if status == 'ok' else 0").field_refs(), {'status'}) + + def test_no_refs(self): + self.assertEqual(Expr('1 + 2').field_refs(), set()) + + +class ExprValidationTest(unittest.TestCase): + """Tests for expression validation / rejection.""" + + def test_rejects_import(self): + with self.assertRaises((ValueError, SyntaxError)): + Expr('__import__("os")') + + def test_rejects_unknown_function(self): + with self.assertRaises(ValueError) as ctx: + Expr('eval("bad")') + self.assertIn('eval', str(ctx.exception)) + + def test_rejects_attribute_access(self): + with self.assertRaises(ValueError): + Expr('x.__class__') + + def test_rejects_subscript(self): + with self.assertRaises(ValueError): + Expr('x[0]') + + def test_rejects_lambda(self): + with self.assertRaises((ValueError, SyntaxError)): + Expr('lambda x: x') + + def test_rejects_chained_comparison(self): + with self.assertRaises(ValueError) as ctx: + Expr('a < b < c') + self.assertIn('Chained', str(ctx.exception)) + + def test_rejects_keyword_args(self): + with self.assertRaises(ValueError) as ctx: + Expr('round(x, ndigits=2)') + self.assertIn('Keyword', str(ctx.exception)) + + def test_rejects_unsupported_literal(self): + with self.assertRaises(ValueError) as ctx: + Expr('b"bytes"') + self.assertIn('bytes', str(ctx.exception)) + + +class ExprPickleTest(unittest.TestCase): + """Tests for pickle/unpickle support.""" + + def test_roundtrip(self): + import pickle + original = Expr('a + b') + restored = pickle.loads(pickle.dumps(original)) + self.assertEqual(original, restored) + self.assertEqual(restored({'a': 1, 'b': 2}), 3) + + +class ExprRealWorldTest(unittest.TestCase): + """Tests using realistic metric expressions.""" + + def test_ctr(self): + e = Expr('clicks / impressions') + self.assertAlmostEqual(e({'clicks': 50, 'impressions': 1000}), 0.05) + + def test_safe_ctr(self): + e = Expr('clicks / max(impressions, 1)') + self.assertEqual(e({'clicks': 0, 'impressions': 0}), 0.0) + + def test_derived_field(self): + e = Expr("1 if status == 'success' else 0") + self.assertEqual(e({'status': 'success'}), 1) + self.assertEqual(e({'status': 'error'}), 0) + + def test_threshold_expression(self): + e = Expr('value >= 100') + self.assertTrue(e({'value': 500})) + self.assertFalse(e({'value': 50})) + + def test_range_threshold(self): + e = Expr('value > 100 or value < -100') + self.assertTrue(e({'value': 200})) + self.assertTrue(e({'value': -200})) + self.assertFalse(e({'value': 50})) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_fanout_windowing.py b/test_fanout_windowing.py new file mode 100644 index 0000000000..4b6fc3aadb --- /dev/null +++ b/test_fanout_windowing.py @@ -0,0 +1,419 @@ +""" +Test to validate windowing behavior of SHARDED vs HOTKEY_FANOUT approaches +with both FixedWindows and SlidingWindows. + +Demonstrates: +1. Both approaches produce correct results with FixedWindows +2. HOTKEY_FANOUT raises ValueError with SlidingWindows (Beam guard) +3. SHARDED works correctly with SlidingWindows +4. Manual hotkey reproduction shows the actual corruption + +ROOT CAUSE OF CORRUPTION (core.py:3349): + "Avoid double counting that may happen with stacked accumulating mode." + +Beam's with_hot_key_fanout splits into hot and cold branches, then Flattens +them before a final CombinePerKey. The two WindowInto steps exist to prevent +accumulating-mode double-counting at the Flatten: + + WindowInto(DISCARDING) → pre-combine GBK → StripNonce → WindowInto(original) → Flatten → final GBK + +But WindowInto(SlidingWindows) re-evaluates window assignments from timestamps, +causing accumulators to leak into adjacent overlapping windows. + +SHARDED avoids this entirely: no hot/cold split, no Flatten, no WindowInto. +Every element takes the same path through both GBKs with window objects +preserved unchanged by Map operations. + +Run: + source venv/bin/activate && python test_fanout_windowing.py +""" +import random + +import apache_beam as beam +from apache_beam import CombineFn +from apache_beam.options.pipeline_options import PipelineOptions +from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.util import assert_that, equal_to +from apache_beam.transforms.window import ( + FixedWindows, SlidingWindows, TimestampedValue) + +# Disable type checking so TimestampedValue doesn't cause issues +TEST_OPTIONS = PipelineOptions(['--no_pipeline_type_check']) + + +class SumCombineFn(CombineFn): + def create_accumulator(self): + return 0 + + def add_input(self, accumulator, element): + return accumulator + element + + def merge_accumulators(self, accumulators): + return sum(accumulators) + + def extract_output(self, accumulator): + return accumulator + + +# --------------------------------------------------------------------------- +# Two-stage CombineFn helpers (same pattern as bqmonitor _PreCombineFn / +# _PostCombineFn and Beam's own PreCombineFn / PostCombineFn in core.py) +# --------------------------------------------------------------------------- + +class PreCombineFn(CombineFn): + """Stage 1: extract_output returns raw accumulator, not final value. + + This is critical for combiners like Mean where acc=(sum, count) but + output=sum/count. You can merge accumulators but not merge outputs. + """ + def __init__(self, combine_fn): + self._combine_fn = combine_fn + + def create_accumulator(self): + return self._combine_fn.create_accumulator() + + def add_input(self, accumulator, element): + return self._combine_fn.add_input(accumulator, element) + + def merge_accumulators(self, accumulators): + return self._combine_fn.merge_accumulators(accumulators) + + def extract_output(self, accumulator): + return accumulator # raw accumulator, not final output + + +class PostCombineFn(CombineFn): + """Stage 2: add_input merges an accumulator from stage 1.""" + def __init__(self, combine_fn): + self._combine_fn = combine_fn + + def create_accumulator(self): + return self._combine_fn.create_accumulator() + + def add_input(self, accumulator, element): + return self._combine_fn.merge_accumulators([accumulator, element]) + + def merge_accumulators(self, accumulators): + return self._combine_fn.merge_accumulators(accumulators) + + def extract_output(self, accumulator): + return self._combine_fn.extract_output(accumulator) + + +# --------------------------------------------------------------------------- +# Test data: 6 elements, 2 keys (A, B), various timestamps +# +# SlidingWindows(size=10, period=5) assigns each element to 2 windows: +# (A,1) t=1 -> [-5,5), [0,10) +# (A,2) t=2 -> [-5,5), [0,10) +# (B,3) t=7 -> [0,10), [5,15) +# (A,4) t=4 -> [-5,5), [0,10) +# (B,5) t=3 -> [-5,5), [0,10) +# (B,6) t=8 -> [0,10), [5,15) +# +# Expected SUM per (key, window): +# [-5, 5): A=1+2+4=7, B=5 +# [0, 10): A=1+2+4=7, B=3+5+6=14 +# [5, 15): B=3+6=9 +# --------------------------------------------------------------------------- + +TEST_DATA = [ + TimestampedValue(('A', 1), 1.0), + TimestampedValue(('A', 2), 2.0), + TimestampedValue(('B', 3), 7.0), # t=7 matters for sliding windows + TimestampedValue(('A', 4), 4.0), + TimestampedValue(('B', 5), 3.0), + TimestampedValue(('B', 6), 8.0), +] + + +class FormatResult(beam.DoFn): + """Emit (key, value, window_start, window_end) for assertion.""" + def process(self, element, window=beam.DoFn.WindowParam): + key, value = element + yield (key, value, float(window.start), float(window.end)) + + +SLIDING_EXPECTED = [ + ('A', 7, -5.0, 5.0), + ('B', 5, -5.0, 5.0), + ('A', 7, 0.0, 10.0), + ('B', 14, 0.0, 10.0), + ('B', 9, 5.0, 15.0), +] + +FIXED_EXPECTED = [ + ('A', 7, 0.0, 10.0), + ('B', 14, 0.0, 10.0), +] + + +# =================================================================== +# TESTS: FixedWindows (all approaches should work) +# =================================================================== + +def test_fixed_plain(): + """Baseline: plain CombinePerKey with FixedWindows.""" + print('\n=== TEST: FixedWindows + Plain CombinePerKey ===') + with TestPipeline(options=TEST_OPTIONS) as p: + results = ( + p + | beam.Create(TEST_DATA) + | beam.WindowInto(FixedWindows(10)) + | beam.CombinePerKey(sum) + | beam.ParDo(FormatResult()) + ) + assert_that(results, equal_to(FIXED_EXPECTED)) + print('PASSED') + + +def test_fixed_sharded(): + """SHARDED with FixedWindows — two GBKs, no WindowInto between them.""" + print('\n=== TEST: FixedWindows + SHARDED ===') + combine_fn = SumCombineFn() + _num_shards = 2 + + with TestPipeline(options=TEST_OPTIONS) as p: + results = ( + p + | beam.Create(TEST_DATA) + | beam.WindowInto(FixedWindows(10)) + # ShardKey: Map preserves windows, adds random shard prefix + | 'ShardKey' >> beam.Map( + lambda kv: ((random.randint(0, _num_shards - 1), kv[0]), kv[1])) + # Pre-combine: GBK groups by (shard, key, window) + | 'PartialCombine' >> beam.CombinePerKey(PreCombineFn(combine_fn)) + # RestoreKey: Map preserves windows, strips shard prefix + | 'RestoreKey' >> beam.MapTuple(lambda ck, acc: (ck[1], acc)) + # NO WindowInto here — windows flow through unchanged + # Final combine: GBK groups by (key, window), merges accumulators + | 'FinalCombine' >> beam.CombinePerKey(PostCombineFn(combine_fn)) + | beam.ParDo(FormatResult()) + ) + assert_that(results, equal_to(FIXED_EXPECTED)) + print('PASSED') + + +def test_fixed_hotkey(): + """HOTKEY_FANOUT with FixedWindows — Beam's built-in two-stage approach.""" + print('\n=== TEST: FixedWindows + HOTKEY_FANOUT ===') + with TestPipeline(options=TEST_OPTIONS) as p: + results = ( + p + | beam.Create(TEST_DATA) + | beam.WindowInto(FixedWindows(10)) + # Beam's built-in: SplitHotCold → WindowInto(DISCARDING) → + # CombinePerKey(Pre) → StripNonce → WindowInto(original) → + # Flatten(hot, cold) → CombinePerKey(Post) + # + # WindowInto(FixedWindows) is idempotent (each timestamp maps to + # exactly one window), so no corruption occurs. + | beam.CombinePerKey(sum).with_hot_key_fanout(2) + | beam.ParDo(FormatResult()) + ) + assert_that(results, equal_to(FIXED_EXPECTED)) + print('PASSED') + + +# =================================================================== +# TESTS: SlidingWindows (hotkey fanout should fail) +# =================================================================== + +def test_sliding_plain(): + """Baseline: plain CombinePerKey with SlidingWindows.""" + print('\n=== TEST: SlidingWindows + Plain CombinePerKey ===') + with TestPipeline(options=TEST_OPTIONS) as p: + results = ( + p + | beam.Create(TEST_DATA) + | beam.WindowInto(SlidingWindows(10, 5)) + | beam.CombinePerKey(sum) + | beam.ParDo(FormatResult()) + ) + assert_that(results, equal_to(SLIDING_EXPECTED)) + print('PASSED') + + +def test_sliding_sharded(): + """SHARDED with SlidingWindows — works because no WindowInto between GBKs. + + Data flow: + ShardKey(Map) → ((shard, key), value) [windows preserved] + CombinePerKey(Pre) → ((shard, key), acc) [GBK groups by (shard,key,window)] + RestoreKey(Map) → (key, acc) [windows preserved — no WindowInto!] + CombinePerKey(Post) → (key, result) [GBK groups by (key,window)] + + Each Map preserves window objects unchanged. No re-evaluation of window + assignments from timestamps. Elements stay in their correct windows. + """ + print('\n=== TEST: SlidingWindows + SHARDED ===') + combine_fn = SumCombineFn() + _num_shards = 2 + + with TestPipeline(options=TEST_OPTIONS) as p: + results = ( + p + | beam.Create(TEST_DATA) + | beam.WindowInto(SlidingWindows(10, 5)) + | 'ShardKey' >> beam.Map( + lambda kv: ((random.randint(0, _num_shards - 1), kv[0]), kv[1])) + | 'PartialCombine' >> beam.CombinePerKey(PreCombineFn(combine_fn)) + | 'RestoreKey' >> beam.MapTuple(lambda ck, acc: (ck[1], acc)) + | 'FinalCombine' >> beam.CombinePerKey(PostCombineFn(combine_fn)) + | beam.ParDo(FormatResult()) + ) + assert_that(results, equal_to(SLIDING_EXPECTED)) + print('PASSED') + + +def test_sliding_hotkey_raises(): + """HOTKEY_FANOUT with SlidingWindows — Beam raises ValueError. + + The guard exists because WindowInto(SlidingWindows) after the pre-combine + re-evaluates window assignments from timestamps. With overlapping windows, + an accumulator produced for Window[0,10) at timestamp 9.999 would be + re-assigned to BOTH Window[0,10) AND Window[5,15), leaking data. + """ + print('\n=== TEST: SlidingWindows + HOTKEY_FANOUT (expect ValueError) ===') + try: + with TestPipeline(options=TEST_OPTIONS) as p: + results = ( + p + | beam.Create(TEST_DATA) + | beam.WindowInto(SlidingWindows(10, 5)) + | beam.CombinePerKey(sum).with_hot_key_fanout(2) + | beam.ParDo(FormatResult()) + ) + assert_that(results, equal_to(SLIDING_EXPECTED)) + print('UNEXPECTED: Pipeline succeeded without error') + except ValueError as e: + if 'SlidingWindows' in str(e): + print(f'CONFIRMED: Beam raises ValueError: {e}') + else: + print(f'UNEXPECTED ValueError: {e}') + except Exception as e: + print(f'Other error: {type(e).__name__}: {e}') + + +# --------------------------------------------------------------------------- +# Manual reproduction: bypass Beam's guard to observe actual corruption +# --------------------------------------------------------------------------- + +def test_sliding_manual_hotkey_corruption(): + """Manually reproduce hotkey fanout to show the actual corruption. + + This bypasses Beam's ValueError guard by manually implementing the + hotkey fanout pipeline shape: + + ShardWithNonce → CombinePerKey(Pre) → StripNonce → WindowInto(Sliding) → CombinePerKey(Post) + + The corruption happens at the WindowInto step. Example for key B: + + Pre-combine produces: + (B, acc=14) for Window[0,10) with output timestamp=9.999 + + WindowInto(SlidingWindows(10, 5)) re-assigns from timestamp: + t=9.999 → Window[0,10) AND Window[5,15) + ^^^^^^^^^^ + LEAKED! acc=14 was only for [0,10) + + Final combine for Window[5,15) now merges: + acc=14 (leaked from [0,10)) + acc=9 (correct) = 23 ← WRONG, should be 9 + + The reason WindowInto exists at all: Beam's hotkey splits into hot + cold + branches, then Flattens them before a final CombinePerKey. Without + WindowInto(DISCARDING) on the hot path, accumulating-mode triggers would + cause double-counting at the Flatten point. But WindowInto with + SlidingWindows causes this different corruption (window leaking). + """ + print('\n=== TEST: SlidingWindows + Manual HOTKEY (no guard) ===') + print("Bypasses Beam's ValueError to show actual corruption.") + + combine_fn = SumCombineFn() + _num_shards = 2 + + class ShardWithNonce(beam.DoFn): + """Replicates Beam's SplitHotCold nonce-per-bundle sharding.""" + def start_bundle(self): + self._nonce = int(random.getrandbits(31)) + + def process(self, element): + key, value = element + yield ((self._nonce % _num_shards, key), value) + + try: + with TestPipeline(options=TEST_OPTIONS) as p: + results = ( + p + | beam.Create(TEST_DATA) + | beam.WindowInto(SlidingWindows(10, 5)) + | 'Shard' >> beam.ParDo(ShardWithNonce()) + | 'PreCombine' >> beam.CombinePerKey(PreCombineFn(combine_fn)) + | 'StripNonce' >> beam.MapTuple(lambda ck, acc: (ck[1], acc)) + # THIS IS WHERE CORRUPTION HAPPENS: + # WindowInto re-evaluates window assignments from timestamps. + # With SlidingWindows, timestamps map to multiple overlapping + # windows, so accumulators leak into adjacent windows. + | 'Rewindow' >> beam.WindowInto(SlidingWindows(10, 5)) + | 'FinalCombine' >> beam.CombinePerKey(PostCombineFn(combine_fn)) + | 'Format' >> beam.ParDo(FormatResult()) + ) + assert_that(results, equal_to(SLIDING_EXPECTED)) + + print('NO CORRUPTION detected (DirectRunner may not reproduce the issue ' + 'due to single-bundle execution)') + + except Exception as e: + err = str(e) + # Extract the BeamAssertException message from the stack trace + if 'BeamAssertException' in err: + # Find the actual assertion diff + for line in err.split('\n'): + if 'unexpected elements' in line: + print(f'CORRUPTION DETECTED: {line.strip()}') + break + else: + print(f'CORRUPTION DETECTED (see full output for details)') + # Show what went wrong + for line in err.split('\n'): + if line.strip().startswith(('(\'A', '(\'B')): + print(f' {line.strip()}') + else: + print(f'Error: {type(e).__name__}: {e}') + + +# =================================================================== +# Run all tests +# =================================================================== + +if __name__ == '__main__': + print('=' * 70) + print('FANOUT WINDOWING VALIDATION TESTS') + print('=' * 70) + print() + print('Tests validate that SHARDED fanout (composite key approach) works') + print('correctly with all window types, while Beam\'s built-in') + print('with_hot_key_fanout breaks with SlidingWindows.') + print() + print('Root cause: with_hot_key_fanout uses WindowInto between its two') + print('GBKs to prevent accumulating-mode double-counting at a Flatten.') + print('WindowInto with SlidingWindows re-evaluates window assignments,') + print('causing accumulators to leak into adjacent overlapping windows.') + print('SHARDED has no Flatten, so no WindowInto is needed.') + + # Fixed windows — all three should produce correct results + test_fixed_plain() + test_fixed_sharded() + test_fixed_hotkey() + + # Sliding windows — plain and sharded should work, hotkey should fail + test_sliding_plain() + test_sliding_sharded() + test_sliding_hotkey_raises() + + # Manual reproduction of corruption + test_sliding_manual_hotkey_corruption() + + print('\n' + '=' * 70) + print('ALL TESTS COMPLETE') + print('=' * 70) diff --git a/v1/logs.txt b/v1/logs.txt new file mode 100644 index 0000000000..f91c94d1b6 --- /dev/null +++ b/v1/logs.txt @@ -0,0 +1,110 @@ +INFO 2026-03-09T03:39:53.776630163Z Control channel established. +INFO 2026-03-09T03:39:53.780595Z Received WorkerStatus stream from SDK harness sdk-0-0_sibling_1 +INFO 2026-03-09T03:39:53.780915498Z Initializing SDKHarness with unbounded number of workers. +INFO 2026-03-09T03:39:53.783929Z Received WorkerStatus stream from SDK harness sdk-0-0 +INFO 2026-03-09T03:39:53.784362077Z Initializing SDKHarness with unbounded number of workers. +INFO 2026-03-09T03:39:53.786175251Z Python sdk harness starting. +INFO 2026-03-09T03:39:53.787254Z Received Control stream from SDK harness sdk-0-0_sibling_1 in environment ref_Environment_default_environment_1 +INFO 2026-03-09T03:39:53.789668560Z Python sdk harness starting. +INFO 2026-03-09T03:39:53.790370Z Received Control stream from SDK harness sdk-0-0 in environment ref_Environment_default_environment_1 +INFO 2026-03-09T03:39:54.181697Z Initial healthz success +INFO 2026-03-09T03:39:54.777374Z All SDK Harnesses registered! +INFO 2026-03-09T03:39:54.777388Z Enabling element sampling: false +INFO 2026-03-09T03:39:54.777394Z Enabling exception sampling: true +INFO 2026-03-09T03:39:54.777395Z Samples will be placed in gs://dataflow-staging-us-central1-672035346122/tmp/bq-anomaly-detection-20260308-233319.1773027387.496692 +INFO 2026-03-09T03:39:54.777975Z Starting data sampling telemetry fiber to report back every 1m +INFO 2026-03-09T03:39:54.920366824Z All workers have finished the startup processes and began to receive work requests. +INFO 2026-03-09T03:39:54.980275Z ******************************************* +INFO 2026-03-09T03:39:54.980286Z Starting Windmill Service user worker +INFO 2026-03-09T03:39:54.980288Z over GRPC with dispatcher endpoint: +INFO 2026-03-09T03:39:54.980289Z us-central1-dataflowstreaming-pa.googleapis.com +INFO 2026-03-09T03:39:54.980290Z and job_header: +INFO 2026-03-09T03:39:54.980290Z go/debugonly job_id: "2026-03-08_20_33_20-8665938784994540780" project_id: "dataflow-twest" worker_id: "bq-anomaly-detection-2026-03082036-zoa2-harness-df7k" client_id: 5109655414601672542 region_id: "us-central1" +INFO 2026-03-09T03:39:54.980948Z ******************************************* +INFO 2026-03-09T03:39:54.981554Z Experimental GetWorkClient enabled: false for job: 2026-03-08_20_33_20-8665938784994540780 +INFO 2026-03-09T03:39:55.262806Z Experimental GetWorkClient enabled: false for job: 2026-03-08_20_33_20-8665938784994540780 +INFO 2026-03-09T03:39:55.262892Z Using the fiber scheduler with admission limit of 24 +INFO 2026-03-09T03:39:55.262895Z Using the fiber scheduler +INFO 2026-03-09T03:39:55.263112Z Using --util_time_backoff_seed=391544512 +INFO 2026-03-09T03:39:55.264495Z Channel 0x71b3ff5f90a0 state changed to 1 from 0 +INFO 2026-03-09T03:39:55.274121Z Channel 0x71b3ff5f90a0 state changed to 2 from 1 +INFO 2026-03-09T03:40:01.899818Z Network interface setup disabled, skipping... +INFO 2026-03-09T03:40:01.903546Z Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T03:40:14.637564Z found namespace: k8s.io +INFO 2026-03-09T03:40:14.650623Z found namespace: moby +INFO 2026-03-09T03:40:14.651531Z Executing: /bin/lsblk /dev/sda -b -d -J +INFO 2026-03-09T03:40:29.616279Z Successfully sampled resources +INFO 2026-03-09T03:40:54.388741493Z Creating insecure state channel for localhost:12371. +INFO 2026-03-09T03:40:54.389101982Z State channel established. +INFO 2026-03-09T03:40:54.390314Z Received State stream from SDK harness sdk-0-0 +INFO 2026-03-09T03:40:54.390354Z State service got sdk harness for worker sdk-0-0 +INFO 2026-03-09T03:40:54.391253709Z Creating client data channel for localhost:12371 +INFO 2026-03-09T03:40:54.391568660Z Data channel established. +INFO 2026-03-09T03:40:54.392774Z Received Data stream from SDK harness sdk-0-0 +ERROR 2026-03-09T03:40:54.399064064Z [severity: ERROR] Error processing instruction process_bundle-1-12. Original traceback is Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 316, in _execute response = task() ^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 390, in lambda: self.create_worker().do_instruction(request), request) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 669, in do_instruction return getattr(self, request_type)( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 701, in process_bundle bundle_processor = self.bundle_processor_cache.get( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 517, in get processor = bundle_processor.BundleProcessor( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1143, in __init__ self.ops = future.result(timeout=3600) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 456, in result return self.__get_result() ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result raise self._exception File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run result = self.fn(*self.args, **self.kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1218, in create_execution_tree return collections.OrderedDict([( ^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1221, in get_operation(transform_id))) for transform_id in sorted( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1043, in wrapper result = cache[args] = func(*args) ^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py… +DEBUG 2026-03-09T03:40:54.399191551Z Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 316, in _execute response = task() ^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 390, in lambda: self.create_worker().do_instruction(request), request) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 669, in do_instruction return getattr(self, request_type)( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 701, in process_bundle bundle_processor = self.bundle_processor_cache.get( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 517, in get processor = bundle_processor.BundleProcessor( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1143, in __init__ self.ops = future.result(timeout=3600) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 456, in result return self.__get_result() ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result raise self._exception File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run result = self.fn(*self.args, **self.kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1218, in create_execution_tree return collections.OrderedDict([( ^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1221, in get_operation(transform_id))) for transform_id in sorted( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1043, in wrapper result = cache[args] = func(*args) ^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1195, in get_operation transform_consumers = { … +ERROR 2026-03-09T03:40:54.400573Z [severity: ERROR] Work item for sharding key f82d1fc396533601 tokens (380075396200816357, 14268627245424784485) encountered error during processing, will be retried (possibly on another worker): generic::unknown: Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 316, in _execute response = task() ^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 390, in lambda: self.create_worker().do_instruction(request), request) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 669, in do_instruction return getattr(self, request_type)( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 701, in process_bundle bundle_processor = self.bundle_processor_cache.get( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 517, in get processor = bundle_processor.BundleProcessor( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1143, in __init__ self.ops = future.result(timeout=3600) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 456, in result return self.__get_result() ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result raise self._exception File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run result = self.fn(*self.args, **self.kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1218, in create_execution_tree return collections.OrderedDict([( ^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1221, in get_operation(transform_id))) for transform_id in sorted( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1043, in wrapper result = cache[args] = func(*args) … +INFO 2026-03-09T03:40:54.400755Z E0309 03:40:54.400573 87 work_item_processor.cc:587] Work item for sharding key f82d1fc396533601 tokens (380075396200816357, 14268627245424784485) encountered error during processing, will be retried (possibly on another worker): generic::unknown: Traceback (most recent call last): +INFO 2026-03-09T03:40:54.400936Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 316, in _execute +INFO 2026-03-09T03:40:54.400970Z response = task() +INFO 2026-03-09T03:40:54.400989Z ^^^^^^ +INFO 2026-03-09T03:40:54.401005Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 390, in +INFO 2026-03-09T03:40:54.401028Z lambda: self.create_worker().do_instruction(request), request) +INFO 2026-03-09T03:40:54.401049Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401072Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 669, in do_instruction +INFO 2026-03-09T03:40:54.401097Z return getattr(self, request_type)( +INFO 2026-03-09T03:40:54.401113Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401129Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 701, in process_bundle +INFO 2026-03-09T03:40:54.401153Z bundle_processor = self.bundle_processor_cache.get( +INFO 2026-03-09T03:40:54.401171Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401186Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 517, in get +INFO 2026-03-09T03:40:54.401204Z processor = bundle_processor.BundleProcessor( +INFO 2026-03-09T03:40:54.401219Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401234Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1143, in __init__ +INFO 2026-03-09T03:40:54.401252Z self.ops = future.result(timeout=3600) +INFO 2026-03-09T03:40:54.401266Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +WARNING 2026-03-09T03:40:54.401278Z [severity: WARNING] Error while processing a work item: UNKNOWN: Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 316, in _execute response = task() ^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 390, in lambda: self.create_worker().do_instruction(request), request) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 669, in do_instruction return getattr(self, request_type)( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 701, in process_bundle bundle_processor = self.bundle_processor_cache.get( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/sdk_worker.py", line 517, in get processor = bundle_processor.BundleProcessor( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1143, in __init__ self.ops = future.result(timeout=3600) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 456, in result return self.__get_result() ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result raise self._exception File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run result = self.fn(*self.args, **self.kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1218, in create_execution_tree return collections.OrderedDict([( ^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1221, in get_operation(transform_id))) for transform_id in sorted( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1043, in wrapper result = cache[args] = func(*args) ^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1195, in get_operat… +INFO 2026-03-09T03:40:54.401281Z File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 456, in result +INFO 2026-03-09T03:40:54.401302Z return self.__get_result() +INFO 2026-03-09T03:40:54.401317Z ^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401330Z File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result +INFO 2026-03-09T03:40:54.401405Z raise self._exception +INFO 2026-03-09T03:40:54.401428Z File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run +INFO 2026-03-09T03:40:54.401447Z result = self.fn(*self.args, **self.kwargs) +INFO 2026-03-09T03:40:54.401463Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401481Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1218, in create_execution_tree +INFO 2026-03-09T03:40:54.401517Z return collections.OrderedDict([( +INFO 2026-03-09T03:40:54.401533Z ^^ +INFO 2026-03-09T03:40:54.401548Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1221, in +INFO 2026-03-09T03:40:54.401568Z get_operation(transform_id))) for transform_id in sorted( +INFO 2026-03-09T03:40:54.401585Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401602Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1043, in wrapper +INFO 2026-03-09T03:40:54.401625Z result = cache[args] = func(*args) +INFO 2026-03-09T03:40:54.401642Z ^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401676Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1195, in get_operation +INFO 2026-03-09T03:40:54.401700Z transform_consumers = { +INFO 2026-03-09T03:40:54.401716Z ^ +INFO 2026-03-09T03:40:54.401740Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1196, in +INFO 2026-03-09T03:40:54.401759Z tag: [get_operation(op) for op in pcoll_consumers[pcoll_id]] +INFO 2026-03-09T03:40:54.401770Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401780Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1196, in +INFO 2026-03-09T03:40:54.401791Z tag: [get_operation(op) for op in pcoll_consumers[pcoll_id]] +INFO 2026-03-09T03:40:54.401801Z ^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401810Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1043, in wrapper +INFO 2026-03-09T03:40:54.401823Z result = cache[args] = func(*args) +INFO 2026-03-09T03:40:54.401832Z ^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401840Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1195, in get_operation +INFO 2026-03-09T03:40:54.401852Z transform_consumers = { +INFO 2026-03-09T03:40:54.401872Z ^ +INFO 2026-03-09T03:40:54.401881Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1196, in +INFO 2026-03-09T03:40:54.401893Z tag: [get_operation(op) for op in pcoll_consumers[pcoll_id]] +INFO 2026-03-09T03:40:54.401902Z ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401912Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1196, in +INFO 2026-03-09T03:40:54.401927Z tag: [get_operation(op) for op in pcoll_consumers[pcoll_id]] +INFO 2026-03-09T03:40:54.401937Z ^^^^^^^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401945Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1043, in wrapper +INFO 2026-03-09T03:40:54.401957Z result = cache[args] = func(*args) +INFO 2026-03-09T03:40:54.401966Z ^^^^^^^^^^^ +INFO 2026-03-09T03:40:54.401975Z File "/usr/local/lib/python3.11/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1195, in get_operation +INFO 2026-03-09T03:40:54.401993Z transform_consumers = { +INFO 2026-03-09T03:40:54.402002Z ^ diff --git a/v1/src/main/java/com/google/cloud/teleport/templates/unwrap-hec-payload.js b/v1/src/main/java/com/google/cloud/teleport/templates/unwrap-hec-payload.js new file mode 100644 index 0000000000..928f8a2b40 --- /dev/null +++ b/v1/src/main/java/com/google/cloud/teleport/templates/unwrap-hec-payload.js @@ -0,0 +1,51 @@ +/** + * Unwraps Splunk HEC-formatted payloads to prevent event nesting. + * + * This function detects if the input is a HEC-formatted JSON object + * (containing an "event" field) and extracts just the event content. + * If the input is not HEC-formatted, it returns it unchanged. + * + * This prevents event nesting when replaying messages from deadletter queue. + * + * @param {string} inJson - Input JSON string (may be HEC-formatted or regular) + * @return {string} - Unwrapped payload + */ +function transform(inJson) { + try { + var obj = JSON.parse(inJson); + + // Check if this looks like HEC format + // HEC format has "event" field and may have other metadata fields + if (obj.hasOwnProperty('event')) { + + // List of known HEC metadata fields + var hecFields = ['event', 'time', 'host', 'index', 'source', 'sourcetype', 'fields']; + var objKeys = Object.keys(obj); + + // Check if all keys are HEC fields (strong indicator it's HEC format) + var isHecFormat = objKeys.every(function(key) { + return hecFields.indexOf(key) !== -1; + }); + + // If it looks like HEC format, extract the event + if (isHecFormat) { + var event = obj.event; + + // The event could be a string or an object + if (typeof event === 'string') { + return event; + } else { + // If event is an object/array, stringify it + return JSON.stringify(event); + } + } + } + + // Not HEC format, return as-is + return inJson; + + } catch (e) { + // If parsing fails, it's not JSON - return as-is + return inJson; + } +} diff --git a/yaml/src/test/python/logs.txt b/yaml/src/test/python/logs.txt new file mode 100644 index 0000000000..34894f73df --- /dev/null +++ b/yaml/src/test/python/logs.txt @@ -0,0 +1,174 @@ +INFO 2026-03-09T02:06:28.673334Z 25da79b966a1: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z bb49a14329f4: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 9da13df7853e: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 97a855169b32: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 4f4fb700ef54: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z ffa033d43a24: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 69f5905c68af: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 1903739d92ce: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z db5da881245a: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z b9b516572f65: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z a163ea7aaf4d: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 15f39d89d5ea: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 37486ddfaacf: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z d9ef0252c0f9: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 454acab13e47: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 9627d305a483: Pulling fs layer +INFO 2026-03-09T02:06:28.673334Z 6dc8224e38b6: Waiting +INFO 2026-03-09T02:06:28.673334Z dca0c57760a4: Waiting +INFO 2026-03-09T02:06:28.673334Z a89a34efd609: Waiting +INFO 2026-03-09T02:06:28.673334Z d513cb1965c3: Waiting +INFO 2026-03-09T02:06:28.673334Z c9e80133c84a: Waiting +INFO 2026-03-09T02:06:28.673334Z 25da79b966a1: Waiting +INFO 2026-03-09T02:06:28.673334Z bb49a14329f4: Waiting +INFO 2026-03-09T02:06:28.673334Z 9da13df7853e: Waiting +INFO 2026-03-09T02:06:28.673334Z 97a855169b32: Waiting +INFO 2026-03-09T02:06:28.673334Z 4f4fb700ef54: Waiting +INFO 2026-03-09T02:06:28.673334Z ffa033d43a24: Waiting +INFO 2026-03-09T02:06:28.673334Z 69f5905c68af: Waiting +INFO 2026-03-09T02:06:28.673334Z 1903739d92ce: Waiting +INFO 2026-03-09T02:06:28.673334Z db5da881245a: Waiting +INFO 2026-03-09T02:06:28.673334Z b9b516572f65: Waiting +INFO 2026-03-09T02:06:28.673334Z a163ea7aaf4d: Waiting +INFO 2026-03-09T02:06:28.673334Z 15f39d89d5ea: Waiting +INFO 2026-03-09T02:06:28.673334Z 37486ddfaacf: Waiting +INFO 2026-03-09T02:06:28.673334Z d9ef0252c0f9: Waiting +INFO 2026-03-09T02:06:28.673334Z 454acab13e47: Waiting +INFO 2026-03-09T02:06:28.673334Z 9627d305a483: Waiting +INFO 2026-03-09T02:06:29.350835Z 89edcaae7ec4: Verifying Checksum +INFO 2026-03-09T02:06:29.350835Z 89edcaae7ec4: Download complete +INFO 2026-03-09T02:06:29.686224Z 51a25504b9e4: Verifying Checksum +INFO 2026-03-09T02:06:29.686224Z 51a25504b9e4: Download complete +INFO 2026-03-09T02:06:30.363201Z bbceb0035429: Verifying Checksum +INFO 2026-03-09T02:06:30.363201Z bbceb0035429: Download complete +INFO 2026-03-09T02:06:30.363201Z dca0c57760a4: Verifying Checksum +INFO 2026-03-09T02:06:30.363201Z dca0c57760a4: Download complete +INFO 2026-03-09T02:06:30.701087Z d513cb1965c3: Verifying Checksum +INFO 2026-03-09T02:06:30.701087Z d513cb1965c3: Download complete +INFO 2026-03-09T02:06:30.701087Z c9e80133c84a: Verifying Checksum +INFO 2026-03-09T02:06:30.701087Z c9e80133c84a: Download complete +INFO 2026-03-09T02:06:31.040142Z a89a34efd609: Verifying Checksum +INFO 2026-03-09T02:06:31.040142Z a89a34efd609: Download complete +INFO 2026-03-09T02:06:31.378400Z 25da79b966a1: Verifying Checksum +INFO 2026-03-09T02:06:31.378400Z 25da79b966a1: Download complete +INFO 2026-03-09T02:06:31.378400Z bb49a14329f4: Verifying Checksum +INFO 2026-03-09T02:06:31.378400Z bb49a14329f4: Download complete +INFO 2026-03-09T02:06:32.053527Z 97a855169b32: Verifying Checksum +INFO 2026-03-09T02:06:32.053527Z 97a855169b32: Download complete +INFO 2026-03-09T02:06:32.053527Z 6dc8224e38b6: Verifying Checksum +INFO 2026-03-09T02:06:32.053527Z 6dc8224e38b6: Download complete +INFO 2026-03-09T02:06:32.053527Z 4f4fb700ef54: Verifying Checksum +INFO 2026-03-09T02:06:32.053527Z 4f4fb700ef54: Download complete +INFO 2026-03-09T02:06:32.393404Z 69f5905c68af: Verifying Checksum +INFO 2026-03-09T02:06:32.393404Z 69f5905c68af: Download complete +INFO 2026-03-09T02:06:32.732299Z 1903739d92ce: Verifying Checksum +INFO 2026-03-09T02:06:32.732299Z 1903739d92ce: Download complete +INFO 2026-03-09T02:06:32.732299Z ffa033d43a24: Verifying Checksum +INFO 2026-03-09T02:06:32.732299Z ffa033d43a24: Download complete +INFO 2026-03-09T02:06:33.076752Z b9b516572f65: Verifying Checksum +INFO 2026-03-09T02:06:33.076752Z b9b516572f65: Download complete +INFO 2026-03-09T02:06:33.416626Z a163ea7aaf4d: Verifying Checksum +INFO 2026-03-09T02:06:33.416626Z a163ea7aaf4d: Download complete +INFO 2026-03-09T02:06:33.416626Z db5da881245a: Verifying Checksum +INFO 2026-03-09T02:06:33.416626Z db5da881245a: Download complete +INFO 2026-03-09T02:06:33.757106Z 15f39d89d5ea: Verifying Checksum +INFO 2026-03-09T02:06:33.757106Z 15f39d89d5ea: Download complete +INFO 2026-03-09T02:06:33.757106Z 37486ddfaacf: Verifying Checksum +INFO 2026-03-09T02:06:33.757106Z 37486ddfaacf: Download complete +INFO 2026-03-09T02:06:34.094939Z 454acab13e47: Verifying Checksum +INFO 2026-03-09T02:06:34.094939Z 454acab13e47: Download complete +INFO 2026-03-09T02:06:37.139572Z 9627d305a483: Verifying Checksum +INFO 2026-03-09T02:06:37.139572Z 9627d305a483: Download complete +INFO 2026-03-09T02:06:37.814366Z d9ef0252c0f9: Verifying Checksum +INFO 2026-03-09T02:06:37.814366Z d9ef0252c0f9: Download complete +DEFAULT 2026-03-09T02:06:38.420039793Z {"logging.googleapis.com/diagnostic":{…}} +INFO 2026-03-09T02:06:39.162925Z 9da13df7853e: Verifying Checksum +INFO 2026-03-09T02:06:39.162925Z 9da13df7853e: Download complete +INFO 2026-03-09T02:06:41.915470Z 51a25504b9e4: Pull complete +INFO 2026-03-09T02:06:42.921627Z 89edcaae7ec4: Pull complete +INFO 2026-03-09T02:06:46.959514Z bbceb0035429: Pull complete +INFO 2026-03-09T02:06:57.091681Z 6dc8224e38b6: Pull complete +INFO 2026-03-09T02:06:57.463157Z dca0c57760a4: Pull complete +INFO 2026-03-09T02:06:58.758697Z a89a34efd609: Pull complete +INFO 2026-03-09T02:06:58.783226Z d513cb1965c3: Pull complete +INFO 2026-03-09T02:06:58.812620Z c9e80133c84a: Pull complete +INFO 2026-03-09T02:06:58.868172Z 25da79b966a1: Pull complete +INFO 2026-03-09T02:06:59.423365Z bb49a14329f4: Pull complete +INFO 2026-03-09T02:07:12.193327114Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:07:12.243909585Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:07:53.309512Z 9da13df7853e: Pull complete +INFO 2026-03-09T02:07:53.496088Z 97a855169b32: Pull complete +INFO 2026-03-09T02:07:53.535016Z 4f4fb700ef54: Pull complete +INFO 2026-03-09T02:07:54.613317Z ffa033d43a24: Pull complete +INFO 2026-03-09T02:07:54.642387Z 69f5905c68af: Pull complete +INFO 2026-03-09T02:07:54.700608Z 1903739d92ce: Pull complete +INFO 2026-03-09T02:07:55.362918Z db5da881245a: Pull complete +INFO 2026-03-09T02:07:55.471125Z b9b516572f65: Pull complete +INFO 2026-03-09T02:07:55.494017Z a163ea7aaf4d: Pull complete +INFO 2026-03-09T02:07:55.529306Z 15f39d89d5ea: Pull complete +INFO 2026-03-09T02:07:55.577133Z 37486ddfaacf: Pull complete +INFO 2026-03-09T02:08:05.403942Z d9ef0252c0f9: Pull complete +INFO 2026-03-09T02:08:05.463627Z 454acab13e47: Pull complete +INFO 2026-03-09T02:08:07.426188Z 9627d305a483: Pull complete +INFO 2026-03-09T02:08:07.499032Z Digest: sha256:33c3234a2972acdfaf6e4e4923c4c04cdfa5386584752b307920ffa7e0e4d81c +INFO 2026-03-09T02:08:07.527300Z Status: Downloaded newer image for gcr.io/dataflow-twest/bigquery-anomaly-detection:templates +INFO 2026-03-09T02:08:10.256159Z Started template launcher. +INFO 2026-03-09T02:08:10.256442Z Created new fluentd log writer for: /var/log/dataflow/template_launcher/runner-json.log +INFO 2026-03-09T02:08:10.272963Z Initialize Python template. +INFO 2026-03-09T02:08:10.272994Z Falling back to using template-container args from metadata: template-container-args +INFO 2026-03-09T02:08:10.274655Z Validating template-container-args: {"consoleLogsLocation":"gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/console_logs","environment":{"additionalUserLabels":{"goog-dataflow-provided-template-name":"bigquery_anomaly_detection","goog-dataflow-provided-template-type":"flex","goog-dataflow-provided-template-version":"templates"},"region":"us-central1","serviceAccountEmail":"672035346122-compute@developer.gserviceaccount.com","stagingLocation":"gs://dataflow-staging-us-central1-672035346122/staging","tempLocation":"gs://dataflow-staging-us-central1-672035346122/tmp"},"jobId":"2026-03-08_19_05_44-10851595192709598734","jobName":"bq-anomaly-detection-20260308-220542","jobObjectLocation":"gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/job_object","operationResultLocation":"gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/operation_result","parameters":{"detector_spec":"{\"type\":\"ZScore\"}","duration_sec":"60000","labels":"goog-dataflow-provided-template-name=bigquery_anomaly_detection,goog-dataflow-provided-template-type=flex,goog-dataflow-provided-template-version=templates","metric_spec":"{\"aggregation\":{\"window\":{\"type\":\"fixed\",\"size_seconds\":60},\"measures\":[{\"field\":\"amount\",\"agg\":\"SUM\",\"alias\":\"revenue\"}]}}","staging_location":"gs://dataflow-staging-us-central1-672035346122/staging","table":"dataflow-twest:cdc.cuj1_revenue","temp_location":"gs://dataflow-staging-us-central1-672035346122/tmp"},"projectId":"dataflow-twest","projectNumber":"672035346122"} +INFO 2026-03-09T02:08:10.275196Z Extracting operation result location. +INFO 2026-03-09T02:08:10.275222Z Operation result location: gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/operation_result +INFO 2026-03-09T02:08:10.275243Z Extracting console log location. +INFO 2026-03-09T02:08:10.275258Z Console logs location: gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/console_logs +INFO 2026-03-09T02:08:10.275285Z Extracting Python command specs. +INFO 2026-03-09T02:08:10.275579Z py options not set in env +INFO 2026-03-09T02:08:10.275881Z Generating launch args. +INFO 2026-03-09T02:08:10.275944Z extra package not set in env +INFO 2026-03-09T02:08:10.276149Z Overriding temp_location with value: gs://dataflow-staging-us-central1-672035346122/tmp (previous value: gs://dataflow-staging-us-central1-672035346122/tmp) +INFO 2026-03-09T02:08:10.276191Z Overriding staging_location with value: gs://dataflow-staging-us-central1-672035346122/staging (previous value: gs://dataflow-staging-us-central1-672035346122/staging) +INFO 2026-03-09T02:08:10.276227Z Validating ExpectedFeatures. +INFO 2026-03-09T02:08:10.276256Z Launching Python template. +INFO 2026-03-09T02:08:10.276286Z Using launch args: [main.py --requirements_file=requirements.txt --setup_file=setup.py --service_account_email=672035346122-compute@developer.gserviceaccount.com --temp_location=gs://dataflow-staging-us-central1-672035346122/tmp --duration_sec=60000 --labels=goog-dataflow-provided-template-name=bigquery_anomaly_detection,goog-dataflow-provided-template-type=flex,goog-dataflow-provided-template-version=templates --metric_spec={"aggregation":{"window":{"type":"fixed","size_seconds":60},"measures":[{"field":"amount","agg":"SUM","alias":"revenue"}]}} --job_name=bq-anomaly-detection-20260308-220542 --template_location=gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/job_object --staging_location=gs://dataflow-staging-us-central1-672035346122/staging --detector_spec={"type":"ZScore"} --table=dataflow-twest:cdc.cuj1_revenue --runner=DataflowRunner --project=dataflow-twest --region=us-central1 --labels=goog-dataflow-provided-template-name=bigquery_anomaly_detection --labels=goog-dataflow-provided-template-type=flex --labels=goog-dataflow-provided-template-version=templates] +INFO 2026-03-09T02:08:10.276363Z Executing: python main.py --requirements_file=requirements.txt --setup_file=setup.py --service_account_email=672035346122-compute@developer.gserviceaccount.com --temp_location=gs://dataflow-staging-us-central1-672035346122/tmp --duration_sec=60000 --labels=goog-dataflow-provided-template-name=bigquery_anomaly_detection,goog-dataflow-provided-template-type=flex,goog-dataflow-provided-template-version=templates --metric_spec={"aggregation":{"window":{"type":"fixed","size_seconds":60},"measures":[{"field":"amount","agg":"SUM","alias":"revenue"}]}} --job_name=bq-anomaly-detection-20260308-220542 --template_location=gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/job_object --staging_location=gs://dataflow-staging-us-central1-672035346122/staging --detector_spec={"type":"ZScore"} --table=dataflow-twest:cdc.cuj1_revenue --runner=DataflowRunner --project=dataflow-twest --region=us-central1 --labels=goog-dataflow-provided-template-name=bigquery_anomaly_detection --labels=goog-dataflow-provided-template-type=flex --labels=goog-dataflow-provided-template-version=templates +INFO 2026-03-09T02:08:12.261837790Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:08:12.285837729Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:08:15.368191Z INFO:root:Runner defaulting to pickling library: cloudpickle. +INFO 2026-03-09T02:08:15.699737Z INFO:bqmonitor.pipeline:Anomaly Monitor Pipeline +INFO 2026-03-09T02:08:15.700040Z INFO:bqmonitor.pipeline: Table: dataflow-twest:cdc.cuj1_revenue +INFO 2026-03-09T02:08:15.700801Z INFO:bqmonitor.pipeline: Detector: ZScore +INFO 2026-03-09T02:08:15.700845Z INFO:bqmonitor.pipeline: Poll interval: 60 sec +INFO 2026-03-09T02:08:15.700858Z INFO:bqmonitor.pipeline: Change function: APPENDS +INFO 2026-03-09T02:08:15.703105Z INFO:bqmonitor.cdc:[ReadBigQueryChangeHistory] expand: table=dataflow-twest:cdc.cuj1_revenue, project=dataflow-twest, change_function=APPENDS, poll_interval=60 sec, buffer=15 sec, temp_dataset=beam_ch_temp_d675c22ea835, start_time=2026-03-09T02:07:15.699528, stop_time=2026-03-09T18:48:15.699561 +INFO 2026-03-09T02:08:16.505962Z INFO:apache_beam.runners.portability.stager:Executing command: ['/usr/local/bin/python', '-m', 'pip', 'download', '--dest', '/tmp/dataflow-requirements-cache', '-r', '/tmp/tmpdomyicgq/tmp_requirements.txt', '--exists-action', 'i', '--no-deps', '--implementation', 'cp', '--abi', 'cp311', '--platform', 'manylinux2014_x86_64'] +INFO 2026-03-09T02:09:12.328460761Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:09:12.337164465Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:10:12.399860201Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:10:12.407236450Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:11:12.466812105Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:11:12.473020138Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:12:12.533832015Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:12:12.541077867Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:13:12.601196600Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:13:12.608047107Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:14:12.670822784Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:14:12.681825403Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:15:12.738540131Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:15:12.744628891Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:16:12.804301470Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:16:12.812084112Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:17:12.870803438Z [resource.labels.instanceId: 4363657088992515093] Network interface setup disabled, skipping... +INFO 2026-03-09T02:17:12.878947243Z [resource.labels.instanceId: 4363657088992515093] Completed adding/removing routes for aliases, forwarded IP and target-instance IPs +INFO 2026-03-09T02:18:04.071831868Z Shutting down the Sandbox, launcher-2026030819054410851595192709598734 of type GCE Instance, used for launching. +NOTICE 2026-03-09T02:18:04.172540Z [protoPayload.serviceName: Compute Engine] [protoPayload.methodName: delete] [protoPayload.resourceName: projects/dataflow-twest/zones/us-central1-c/instances/launcher-2026030819054410851595192709598734] [protoPayload.authenticationInfo.principalEmail: service-672035346122@dataflow-service-producer-prod.iam.gserviceaccount.com] audit_log, method: "beta.compute.instances.delete", principal_email: "service-672035346122@dataflow-service-producer-prod.iam.gserviceaccount.com" +INFO 2026-03-09T02:18:06.148112188Z [resource.labels.instanceId: 4363657088992515093] Starting shutdown scripts (version 20250701.01). +DEFAULT 2026-03-09T02:18:06.148157580Z [resource.labels.instanceId: 4363657088992515093] {"logging.googleapis.com/diagnostic":{…}} +INFO 2026-03-09T02:18:06.202566873Z [resource.labels.instanceId: 4363657088992515093] No shutdown scripts to run. +DEFAULT 2026-03-09T02:18:06.223194492Z ping +ERROR 2026-03-09T02:18:06.728407894Z [severity: ERROR] [resource.labels.instanceId: 4363657088992515093] Error watching metadata: context canceled +INFO 2026-03-09T02:18:06.728589111Z [resource.labels.instanceId: 4363657088992515093] GCE Agent Stopped +DEFAULT 2026-03-09T02:18:06.764140739Z ping +NOTICE 2026-03-09T02:18:09.238392758Z [resource.labels.instanceId: 4363657088992515093] {"@type":"type.googleapis.com/cloud_integrity.IntegrityEvent", "bootCounter":"1", "shutdownEvent":{…}} +NOTICE 2026-03-09T02:18:26.499917Z [protoPayload.serviceName: Compute Engine] [protoPayload.methodName: delete] [protoPayload.resourceName: projects/dataflow-twest/zones/us-central1-c/instances/launcher-2026030819054410851595192709598734] [protoPayload.authenticationInfo.principalEmail: service-672035346122@dataflow-service-producer-prod.iam.gserviceaccount.com] audit_log, method: "beta.compute.instances.delete", principal_email: "service-672035346122@dataflow-service-producer-prod.iam.gserviceaccount.com" +INFO 2026-03-09T02:18:26.555009267Z Sandbox, launcher-2026030819054410851595192709598734, stopped. +ERROR 2026-03-09T02:18:26.832731464Z [severity: ERROR] Timeout in polling result file: gs://dataflow-staging-us-central1-672035346122/staging/template_launches/2026-03-08_19_05_44-10851595192709598734/operation_result. Service account: 672035346122-compute@developer.gserviceaccount.com Image URL: gcr.io/dataflow-twest/bigquery-anomaly-detection:templates Troubleshooting guide at https://cloud.google.com/dataflow/docs/guides/troubleshoot-templates#timeout-polling