From ed32fe9077fa1aac3832c3997d1fdf876cad2679 Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 17 Jul 2026 13:51:22 +0530 Subject: [PATCH 1/4] update benchmark pipeline --- .github/workflows/test-runner.yml | 71 +++++++++++++++++++++++++++++ .github/workflows/windows-cmake.yml | 1 + 2 files changed, 72 insertions(+) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index a63dca5b0b..971d84ddf0 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -109,3 +109,74 @@ jobs: checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }} build_shard: 'Core' secrets: inherit + + windows-benchmark-results: + name: Windows-Benchmark (Generate Results Table) + if: | + always() && + github.event_name == 'workflow_dispatch' && + (inputs.run_benchmark_bq == true || inputs.run_benchmark_existing == true) + needs: [pre-flight, windows-benchmark-bq, windows-benchmark-existing] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ needs.pre-flight.outputs.checkout-sha }} + + - uses: google-github-actions/auth@v2 + with: + create_credentials_file: true + credentials_json: ${{ secrets.BUILD_CACHE_KEY }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Download Results from GCS + run: | + mkdir -p benchmark_results + + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/current_bq.txt || true + + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_Core.txt ./benchmark_results/current_core.txt || true + + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/main_bq.txt || true + + - name: Parse Results and Generate Table + run: | + python3 -c " + import os + + def get_time(filepath): + if not os.path.exists(filepath): + return 'File Not Found' + try: + with open(filepath, 'r') as f: + for line in f: + if 'Time taken:' in line: + return line.split('Time taken:')[1].strip() + except Exception as e: + return str(e) + return 'Time not found in output' + + current_core = get_time('./benchmark_results/current_core.txt') + current_bq = get_time('./benchmark_results/current_bq.txt') + main_bq = get_time('./benchmark_results/main_bq.txt') + + table = f\"\"\"| Simba Driver (Current Branch) | Google Driver (Current Branch) | Google Driver (Main Branch) | + |-------------------------------|--------------------------------|-----------------------------| + | {current_core.ljust(29)} | {current_bq.ljust(30)} | {main_bq.ljust(27)} | + \"\"\" + + with open('benchmark_summary_table.txt', 'w') as f: + f.write(table) + + print('Generated Benchmark Table:') + print(table) + " + + - name: Upload Table as Artifact + uses: actions/upload-artifact@v4 + with: + name: benchmark-summary-table + path: benchmark_summary_table.txt \ No newline at end of file diff --git a/.github/workflows/windows-cmake.yml b/.github/workflows/windows-cmake.yml index 1c48d7f009..60f1eb16a7 100644 --- a/.github/workflows/windows-cmake.yml +++ b/.github/workflows/windows-cmake.yml @@ -157,6 +157,7 @@ jobs: reg import $regFilePath Write-Output "DSN creation completed." - name: Running testcases + if: false shell: bash run: | export VCPKG_ROOT="${TEMP}/.build/vcpkg" From 7d3c661a772b1ab43761274a3543870d9c63d562 Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 17 Jul 2026 14:45:08 +0530 Subject: [PATCH 2/4] test --- .github/workflows/test-runner.yml | 47 ++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 971d84ddf0..7c98bd4a85 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -146,32 +146,51 @@ jobs: run: | python3 -c " import os + import re - def get_time(filepath): + def parse_gtest_output(filepath): + results = {} if not os.path.exists(filepath): - return 'File Not Found' + return results + + # Regex to capture the GoogleTest output format: [ OK ] Test.Name (Time) + pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)') try: with open(filepath, 'r') as f: for line in f: - if 'Time taken:' in line: - return line.split('Time taken:')[1].strip() + match = pattern.search(line) + if match: + test_name = match.group(1) + time_taken = match.group(2) + results[test_name] = time_taken except Exception as e: - return str(e) - return 'Time not found in output' + print(f'Error reading {filepath}: {e}') + return results - current_core = get_time('./benchmark_results/current_core.txt') - current_bq = get_time('./benchmark_results/current_bq.txt') - main_bq = get_time('./benchmark_results/main_bq.txt') + # Extract dictionaries mapping TestName -> Time + simba_data = parse_gtest_output('./benchmark_results/current_core.txt') + current_bq_data = parse_gtest_output('./benchmark_results/current_bq.txt') + main_bq_data = parse_gtest_output('./benchmark_results/main_bq.txt') - table = f\"\"\"| Simba Driver (Current Branch) | Google Driver (Current Branch) | Google Driver (Main Branch) | - |-------------------------------|--------------------------------|-----------------------------| - | {current_core.ljust(29)} | {current_bq.ljust(30)} | {main_bq.ljust(27)} | - \"\"\" + # Get a unique, sorted list of all test cases across all three runs + all_tests = set(simba_data.keys()).union(set(current_bq_data.keys())).union(set(main_bq_data.keys())) + sorted_tests = sorted(list(all_tests)) + # Build the Markdown table + table = '| Test Case (HTAPI ON/OFF) | Simba Driver (Current) | Google Driver (Current) | Google Driver (Main) |\n' + table += '|---|---|---|---|\n' + + for test in sorted_tests: + simba_time = simba_data.get(test, 'N/A') + cur_bq_time = current_bq_data.get(test, 'N/A') + main_bq_time = main_bq_data.get(test, 'N/A') + table += f'| {test} | {simba_time} | {cur_bq_time} | {main_bq_time} |\n' + + # Write table to file with open('benchmark_summary_table.txt', 'w') as f: f.write(table) - print('Generated Benchmark Table:') + print('Generated Benchmark Table:\n') print(table) " From 5df947b488f4eba2fda7258de9115796d95ffe65 Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Mon, 20 Jul 2026 09:09:53 +0530 Subject: [PATCH 3/4] making final result readable --- .github/workflows/test-runner.yml | 56 ++++++++++++++++++----------- .github/workflows/windows-cmake.yml | 1 - 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 7c98bd4a85..49740b5c23 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -124,22 +124,22 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ needs.pre-flight.outputs.checkout-sha }} - + - uses: google-github-actions/auth@v2 with: create_credentials_file: true credentials_json: ${{ secrets.BUILD_CACHE_KEY }} - + - uses: google-github-actions/setup-gcloud@v2 - name: Download Results from GCS run: | mkdir -p benchmark_results - + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/current_bq.txt || true - + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/performance_benchmark_results_Core.txt ./benchmark_results/current_core.txt || true - + gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/main_bq.txt || true - name: Parse Results and Generate Table @@ -150,10 +150,9 @@ jobs: def parse_gtest_output(filepath): results = {} - if not os.path.exists(filepath): + if not os.path.exists(filepath): return results - - # Regex to capture the GoogleTest output format: [ OK ] Test.Name (Time) + pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)') try: with open(filepath, 'r') as f: @@ -167,29 +166,46 @@ jobs: print(f'Error reading {filepath}: {e}') return results - # Extract dictionaries mapping TestName -> Time - simba_data = parse_gtest_output('./benchmark_results/current_core.txt') + def clean_test_name(name): + name = name.replace('HTAPIVariations/CatalogPerformanceHtapiParamTest.', '') + name = name.replace('Tables/DataFetchPerformanceParamTest.', '') + return name + + existing_data = parse_gtest_output('./benchmark_results/current_core.txt') current_bq_data = parse_gtest_output('./benchmark_results/current_bq.txt') main_bq_data = parse_gtest_output('./benchmark_results/main_bq.txt') - # Get a unique, sorted list of all test cases across all three runs - all_tests = set(simba_data.keys()).union(set(current_bq_data.keys())).union(set(main_bq_data.keys())) + all_tests = set(existing_data.keys()).union(set(current_bq_data.keys())).union(set(main_bq_data.keys())) sorted_tests = sorted(list(all_tests)) - # Build the Markdown table - table = '| Test Case (HTAPI ON/OFF) | Simba Driver (Current) | Google Driver (Current) | Google Driver (Main) |\n' - table += '|---|---|---|---|\n' - + # Define headers + h1 = 'Test Case (HTAPI ON/OFF)' + h2 = 'Existing Driver (Current)' + h3 = 'Google Driver (Current)' + h4 = 'Google Driver (Main)' + + # Calculate dynamic widths for formatting + w1 = max([len(h1)] + [len(clean_test_name(t)) for t in sorted_tests]) if sorted_tests else len(h1) + w2 = max([len(h2)] + [len(existing_data.get(t, 'N/A')) for t in sorted_tests]) if sorted_tests else len(h2) + w3 = max([len(h3)] + [len(current_bq_data.get(t, 'N/A')) for t in sorted_tests]) if sorted_tests else len(h3) + w4 = max([len(h4)] + [len(main_bq_data.get(t, 'N/A')) for t in sorted_tests]) if sorted_tests else len(h4) + + # Construct the Markdown table + table = f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n' + table += '|-' + ('-' * w1) + '-|-' + ('-' * w2) + '-|-' + ('-' * w3) + '-|-' + ('-' * w4) + '-|\n' + for test in sorted_tests: - simba_time = simba_data.get(test, 'N/A') + cleaned_name = clean_test_name(test) + existing_time = existing_data.get(test, 'N/A') cur_bq_time = current_bq_data.get(test, 'N/A') main_bq_time = main_bq_data.get(test, 'N/A') - table += f'| {test} | {simba_time} | {cur_bq_time} | {main_bq_time} |\n' + + table += f'| {cleaned_name.ljust(w1)} | {existing_time.ljust(w2)} | {cur_bq_time.ljust(w3)} | {main_bq_time.ljust(w4)} |\n' # Write table to file with open('benchmark_summary_table.txt', 'w') as f: f.write(table) - + print('Generated Benchmark Table:\n') print(table) " @@ -198,4 +214,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: benchmark-summary-table - path: benchmark_summary_table.txt \ No newline at end of file + path: benchmark_summary_table.txt diff --git a/.github/workflows/windows-cmake.yml b/.github/workflows/windows-cmake.yml index 60f1eb16a7..1c48d7f009 100644 --- a/.github/workflows/windows-cmake.yml +++ b/.github/workflows/windows-cmake.yml @@ -157,7 +157,6 @@ jobs: reg import $regFilePath Write-Output "DSN creation completed." - name: Running testcases - if: false shell: bash run: | export VCPKG_ROOT="${TEMP}/.build/vcpkg" From 502fa0c4829934824de807077e9ac81819714c2a Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Wed, 22 Jul 2026 13:14:15 +0000 Subject: [PATCH 4/4] chore: updated the perf table to include percentages --- .github/workflows/test-runner.yml | 92 ++++++++++++++----- .../examples/catalog_performance_example.cc | 4 +- 2 files changed, 71 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 49740b5c23..0574ce1848 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -24,6 +24,10 @@ on: description: 'Run Benchmark: Existing Driver' type: boolean default: false + generate_benchmark_results: + description: 'Generate Benchmark Results Table' + type: boolean + default: false # Concurrency configuration: # 1. Automated runs (push events) share the same group, so new commits cancel older in-progress runs. @@ -115,7 +119,7 @@ jobs: if: | always() && github.event_name == 'workflow_dispatch' && - (inputs.run_benchmark_bq == true || inputs.run_benchmark_existing == true) + (inputs.run_benchmark_bq == true || inputs.run_benchmark_existing == true || inputs.generate_benchmark_results == true) needs: [pre-flight, windows-benchmark-bq, windows-benchmark-existing] runs-on: ubuntu-latest permissions: @@ -144,7 +148,7 @@ jobs: - name: Parse Results and Generate Table run: | - python3 -c " + python3 <<'EOF' import os import re @@ -166,6 +170,36 @@ jobs: print(f'Error reading {filepath}: {e}') return results + def parse_time_to_ms(time_str): + if not time_str or time_str == 'N/A': + return None + time_str = time_str.strip() + match = re.match(r'^([\d.]+)\s*(\w+)$', time_str) + if not match: + return None + val = float(match.group(1)) + unit = match.group(2).lower() + if unit == 'ms': + return val + elif unit == 's': + return val * 1000 + elif unit == 'us': + return val / 1000 + elif unit == 'ns': + return val / 1000000 + return val + + def get_percentage_str(val_ms, ref_ms): + if val_ms is None or ref_ms is None or ref_ms == 0: + return ' (N/A)' + pct = round(((val_ms - ref_ms) / ref_ms) * 100) + if pct > 0: + return f' (+{pct}%)' + elif pct < 0: + return f' ({pct}%)' + else: + return ' (0%)' + def clean_test_name(name): name = name.replace('HTAPIVariations/CatalogPerformanceHtapiParamTest.', '') name = name.replace('Tables/DataFetchPerformanceParamTest.', '') @@ -178,6 +212,26 @@ jobs: all_tests = set(existing_data.keys()).union(set(current_bq_data.keys())).union(set(main_bq_data.keys())) sorted_tests = sorted(list(all_tests)) + rows = [] + for test in sorted_tests: + cleaned_name = clean_test_name(test) + + existing_raw = existing_data.get(test, 'N/A') + cur_bq_raw = current_bq_data.get(test, 'N/A') + main_bq_raw = main_bq_data.get(test, 'N/A') + + existing_ms = parse_time_to_ms(existing_raw) + cur_bq_ms = parse_time_to_ms(cur_bq_raw) + main_bq_ms = parse_time_to_ms(main_bq_raw) + + cur_bq_pct = get_percentage_str(cur_bq_ms, existing_ms) if cur_bq_raw != 'N/A' else '' + main_bq_pct = get_percentage_str(main_bq_ms, cur_bq_ms) if main_bq_raw != 'N/A' else '' + + cur_bq_val = f'{cur_bq_raw}{cur_bq_pct}' + main_bq_val = f'{main_bq_raw}{main_bq_pct}' + + rows.append((cleaned_name, existing_raw, cur_bq_val, main_bq_val)) + # Define headers h1 = 'Test Case (HTAPI ON/OFF)' h2 = 'Existing Driver (Current)' @@ -185,33 +239,25 @@ jobs: h4 = 'Google Driver (Main)' # Calculate dynamic widths for formatting - w1 = max([len(h1)] + [len(clean_test_name(t)) for t in sorted_tests]) if sorted_tests else len(h1) - w2 = max([len(h2)] + [len(existing_data.get(t, 'N/A')) for t in sorted_tests]) if sorted_tests else len(h2) - w3 = max([len(h3)] + [len(current_bq_data.get(t, 'N/A')) for t in sorted_tests]) if sorted_tests else len(h3) - w4 = max([len(h4)] + [len(main_bq_data.get(t, 'N/A')) for t in sorted_tests]) if sorted_tests else len(h4) + w1 = max([len(h1)] + [len(r[0]) for r in rows]) if rows else len(h1) + w2 = max([len(h2)] + [len(r[1]) for r in rows]) if rows else len(h2) + w3 = max([len(h3)] + [len(r[2]) for r in rows]) if rows else len(h3) + w4 = max([len(h4)] + [len(r[3]) for r in rows]) if rows else len(h4) # Construct the Markdown table - table = f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n' + table = "*Percentages in **Google Driver (Current)** show change relative to **Existing Driver (Current)**. Percentages in **Google Driver (Main)** show change relative to **Google Driver (Current)**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n" + table += f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n' table += '|-' + ('-' * w1) + '-|-' + ('-' * w2) + '-|-' + ('-' * w3) + '-|-' + ('-' * w4) + '-|\n' - for test in sorted_tests: - cleaned_name = clean_test_name(test) - existing_time = existing_data.get(test, 'N/A') - cur_bq_time = current_bq_data.get(test, 'N/A') - main_bq_time = main_bq_data.get(test, 'N/A') - - table += f'| {cleaned_name.ljust(w1)} | {existing_time.ljust(w2)} | {cur_bq_time.ljust(w3)} | {main_bq_time.ljust(w4)} |\n' + for r in rows: + table += f'| {r[0].ljust(w1)} | {r[1].ljust(w2)} | {r[2].ljust(w3)} | {r[3].ljust(w4)} |\n' # Write table to file with open('benchmark_summary_table.txt', 'w') as f: f.write(table) + EOF - print('Generated Benchmark Table:\n') - print(table) - " - - - name: Upload Table as Artifact - uses: actions/upload-artifact@v4 - with: - name: benchmark-summary-table - path: benchmark_summary_table.txt + - name: Upload Table to GCS + run: | + gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/ + echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/${{ github.ref_name }}/results/benchmark_summary_table.txt" diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc index b76ec41737..83296b0bc4 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc @@ -431,10 +431,10 @@ INSTANTIATE_TEST_SUITE_P( Tables, DataFetchPerformanceParamTest, ::testing::Values( std::make_tuple( - "bigquery-devtools-drivers.kirltest.new_timestamp_table", 100000), + "bigquery-devtools-drivers.kirltest.new_timestamp_table", 200000), std::make_tuple( "bigquery-devtools-drivers.INTEGRATION_TEST_FORMAT.all_bq_types_2", - 100000) + 200000) // TODO: Re-enable this benchmark once HTAPI Arrow supports all data // types. Currently SQLExecDirect fails with: // "[Google][ODBC BigQuery Driver] Internal Error: Unsupported arrow