diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index a63dca5b0b..0574ce1848 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -24,6 +24,10 @@ on: description: 'Run Benchmark: Existing Driver' type: boolean default: false + generate_benchmark_results: + description: 'Generate Benchmark Results Table' + type: boolean + default: false # Concurrency configuration: # 1. Automated runs (push events) share the same group, so new commits cancel older in-progress runs. @@ -109,3 +113,151 @@ jobs: checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }} build_shard: 'Core' secrets: inherit + + windows-benchmark-results: + name: Windows-Benchmark (Generate Results Table) + if: | + always() && + github.event_name == 'workflow_dispatch' && + (inputs.run_benchmark_bq == true || inputs.run_benchmark_existing == true || inputs.generate_benchmark_results == true) + needs: [pre-flight, windows-benchmark-bq, windows-benchmark-existing] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ needs.pre-flight.outputs.checkout-sha }} + + - uses: google-github-actions/auth@v2 + with: + create_credentials_file: true + credentials_json: ${{ secrets.BUILD_CACHE_KEY }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Download Results from GCS + run: | + mkdir -p benchmark_results + + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/current_bq.txt || true + + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_Core.txt ./benchmark_results/current_core.txt || true + + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/main_bq.txt || true + + - name: Parse Results and Generate Table + run: | + python3 <<'EOF' + import os + import re + + def parse_gtest_output(filepath): + results = {} + if not os.path.exists(filepath): + return results + + pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)') + try: + with open(filepath, 'r') as f: + for line in f: + match = pattern.search(line) + if match: + test_name = match.group(1) + time_taken = match.group(2) + results[test_name] = time_taken + except Exception as e: + print(f'Error reading {filepath}: {e}') + return results + + def parse_time_to_ms(time_str): + if not time_str or time_str == 'N/A': + return None + time_str = time_str.strip() + match = re.match(r'^([\d.]+)\s*(\w+)$', time_str) + if not match: + return None + val = float(match.group(1)) + unit = match.group(2).lower() + if unit == 'ms': + return val + elif unit == 's': + return val * 1000 + elif unit == 'us': + return val / 1000 + elif unit == 'ns': + return val / 1000000 + return val + + def get_percentage_str(val_ms, ref_ms): + if val_ms is None or ref_ms is None or ref_ms == 0: + return ' (N/A)' + pct = round(((val_ms - ref_ms) / ref_ms) * 100) + if pct > 0: + return f' (+{pct}%)' + elif pct < 0: + return f' ({pct}%)' + else: + return ' (0%)' + + def clean_test_name(name): + name = name.replace('HTAPIVariations/CatalogPerformanceHtapiParamTest.', '') + name = name.replace('Tables/DataFetchPerformanceParamTest.', '') + return name + + existing_data = parse_gtest_output('./benchmark_results/current_core.txt') + current_bq_data = parse_gtest_output('./benchmark_results/current_bq.txt') + main_bq_data = parse_gtest_output('./benchmark_results/main_bq.txt') + + all_tests = set(existing_data.keys()).union(set(current_bq_data.keys())).union(set(main_bq_data.keys())) + sorted_tests = sorted(list(all_tests)) + + rows = [] + for test in sorted_tests: + cleaned_name = clean_test_name(test) + + existing_raw = existing_data.get(test, 'N/A') + cur_bq_raw = current_bq_data.get(test, 'N/A') + main_bq_raw = main_bq_data.get(test, 'N/A') + + existing_ms = parse_time_to_ms(existing_raw) + cur_bq_ms = parse_time_to_ms(cur_bq_raw) + main_bq_ms = parse_time_to_ms(main_bq_raw) + + cur_bq_pct = get_percentage_str(cur_bq_ms, existing_ms) if cur_bq_raw != 'N/A' else '' + main_bq_pct = get_percentage_str(main_bq_ms, cur_bq_ms) if main_bq_raw != 'N/A' else '' + + cur_bq_val = f'{cur_bq_raw}{cur_bq_pct}' + main_bq_val = f'{main_bq_raw}{main_bq_pct}' + + rows.append((cleaned_name, existing_raw, cur_bq_val, main_bq_val)) + + # Define headers + h1 = 'Test Case (HTAPI ON/OFF)' + h2 = 'Existing Driver (Current)' + h3 = 'Google Driver (Current)' + h4 = 'Google Driver (Main)' + + # Calculate dynamic widths for formatting + w1 = max([len(h1)] + [len(r[0]) for r in rows]) if rows else len(h1) + w2 = max([len(h2)] + [len(r[1]) for r in rows]) if rows else len(h2) + w3 = max([len(h3)] + [len(r[2]) for r in rows]) if rows else len(h3) + w4 = max([len(h4)] + [len(r[3]) for r in rows]) if rows else len(h4) + + # Construct the Markdown table + table = "*Percentages in **Google Driver (Current)** show change relative to **Existing Driver (Current)**. Percentages in **Google Driver (Main)** show change relative to **Google Driver (Current)**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n" + table += f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n' + table += '|-' + ('-' * w1) + '-|-' + ('-' * w2) + '-|-' + ('-' * w3) + '-|-' + ('-' * w4) + '-|\n' + + for r in rows: + table += f'| {r[0].ljust(w1)} | {r[1].ljust(w2)} | {r[2].ljust(w3)} | {r[3].ljust(w4)} |\n' + + # Write table to file + with open('benchmark_summary_table.txt', 'w') as f: + f.write(table) + EOF + + - name: Upload Table to GCS + run: | + gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/ + echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/benchmark_summary_table.txt" diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc index b76ec41737..83296b0bc4 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc @@ -431,10 +431,10 @@ INSTANTIATE_TEST_SUITE_P( Tables, DataFetchPerformanceParamTest, ::testing::Values( std::make_tuple( - "bigquery-devtools-drivers.kirltest.new_timestamp_table", 100000), + "bigquery-devtools-drivers.kirltest.new_timestamp_table", 200000), std::make_tuple( "bigquery-devtools-drivers.INTEGRATION_TEST_FORMAT.all_bq_types_2", - 100000) + 200000) // TODO: Re-enable this benchmark once HTAPI Arrow supports all data // types. Currently SQLExecDirect fails with: // "[Google][ODBC BigQuery Driver] Internal Error: Unsupported arrow