diff --git a/.github/workflows/IntegrationTests.yml b/.github/workflows/IntegrationTests.yml index 31e03b583..80f30c449 100644 --- a/.github/workflows/IntegrationTests.yml +++ b/.github/workflows/IntegrationTests.yml @@ -39,10 +39,92 @@ jobs: make format-check linux-tests: - name: Run tests on Linux + name: Linux Tests needs: format-check runs-on: ubuntu-latest + env: + GEN: ninja + CC: 'ccache gcc' + CXX: 'ccache g++' + CCACHE_DIR: ${{ github.workspace }}/ccache + VCPKG_TARGET_TRIPLET: x64-linux-release + VCPKG_HOST_TRIPLET: x64-linux-release + VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Install Dependencies + run: | + sudo apt-get update -y -q -o=Dpkg::Use-Pty=0 + sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \ + build-essential \ + ccache \ + cmake \ + ninja-build + + - name: Cache Key + id: cache_key + working-directory: ./duckdb + run: | + DUCKDB_VERSION=$(git rev-parse --short HEAD) + KEY="${{ runner.os }}-${{ runner.arch }}-$DUCKDB_VERSION" + echo "value=${KEY}" >> "${GITHUB_OUTPUT}" + + - name: Restore Cache + uses: actions/cache/restore@v5 + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.cache_key.outputs.value }} + + - name: Setup Postgres + uses: ikalnytskyi/action-setup-postgres@v8 + with: + postgres-version: 17 + username: 'postgres' + password: 'postgres' + database: 'postgres' + port: '5432' + + - name: Setup vcpkg + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + + - name: Build extension + run: | + make release + + - name: Save Cache + uses: actions/cache/save@v5 + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.cache_key.outputs.value }} + + - name: Run tests + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_TEST_SLOW: 1 + LOCAL_EXTENSION_REPO: 'build/release/repository' + run: | + source ./create-postgres-tables.sh + psql -d postgresscanner -c "SELECT 42" + make test + + linux-pgbouncer: + name: Linux PgBouncer + needs: linux-tests + runs-on: ubuntu-latest + strategy: matrix: pg_version: [14, 18] @@ -125,12 +207,6 @@ jobs: run: | make release - - name: Save Cache - uses: actions/cache/save@v5 - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.cache_key.outputs.value }} - - name: Setup test data env: PGHOST: localhost @@ -147,8 +223,8 @@ jobs: PGUSER: postgres PGPASSWORD: postgres PGSSLMODE: require - PGSSLROOTCERT: ${{ github.workspace }}/pgbouncer/pgbouncer.crt POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_TEST_SLOW: 1 LOCAL_EXTENSION_REPO: 'build/release/repository' run: | psql -d postgresscanner -c "SELECT * FROM pg_stat_ssl WHERE pid = pg_backend_pid()" @@ -161,11 +237,10 @@ jobs: PGUSER: postgres PGPASSWORD: postgres PGSSLMODE: require - PGSSLROOTCERT: ${{ github.workspace }}/server.crt POSTGRES_TEST_DATABASE_AVAILABLE: 1 + POSTGRES_TEST_SLOW: 1 LOCAL_EXTENSION_REPO: 'build/release/repository' run: | - cp $RUNNER_TEMP/pgdata/server.crt . psql -d postgresscanner -c "SELECT * FROM pg_stat_ssl WHERE pid = pg_backend_pid()" make test @@ -177,9 +252,93 @@ jobs: path: | /var/log/postgresql/pgbouncer.log + linux-relassert: + name: Linux RelAssert + needs: linux-tests + runs-on: ubuntu-latest + + env: + CMAKE_BUILD_PARALLEL_LEVEL: 2 + CC: 'ccache gcc' + CXX: 'ccache g++' + CCACHE_DIR: ${{ github.workspace }}/ccache + VCPKG_TARGET_TRIPLET: x64-linux-release + VCPKG_HOST_TRIPLET: x64-linux-release + VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Install Dependencies + run: | + sudo apt-get update -y -q -o=Dpkg::Use-Pty=0 + sudo apt-get install -y -q -o=Dpkg::Use-Pty=0 \ + build-essential \ + ccache \ + cmake + + - name: Cache Key + id: cache_key + working-directory: ./duckdb + run: | + DUCKDB_VERSION=$(git rev-parse --short HEAD) + KEY="${{ runner.os }}-${{ runner.arch }}-$DUCKDB_VERSION"-relassert + echo "value=${KEY}" >> "${GITHUB_OUTPUT}" + + - name: Restore Cache + uses: actions/cache/restore@v5 + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.cache_key.outputs.value }} + + - name: Setup Postgres + uses: ikalnytskyi/action-setup-postgres@v8 + with: + postgres-version: 18 + username: 'postgres' + password: 'postgres' + database: 'postgres' + port: '5432' + ssl: true + + - name: Setup vcpkg + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35 + + - name: Build extension + run: | + make relassert + cd build + ln -s relassert release + + - name: Save Cache + uses: actions/cache/save@v5 + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.cache_key.outputs.value }} + + - name: Run tests + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: postgres + PGSSLMODE: require + POSTGRES_TEST_DATABASE_AVAILABLE: 1 + LOCAL_EXTENSION_REPO: 'build/relassert/repository' + run: | + source ./create-postgres-tables.sh + psql -d postgresscanner -c "SELECT * FROM pg_stat_ssl WHERE pid = pg_backend_pid()" + make test + windows-tests: - name: Run tests on Windows - needs: format-check + name: Windows Tests + needs: linux-tests runs-on: windows-latest env: diff --git a/scripts/time_tests.py b/scripts/time_tests.py new file mode 100755 index 000000000..b8b586281 --- /dev/null +++ b/scripts/time_tests.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import subprocess +import time +import glob + +def run_and_time_unit_tests(): + """ + Discovers and runs unit tests, records their elapsed time, and returns + a list of test results sorted by elapsed time in descending order. + """ + # Assuming the script is run from the project root (e.g., duckdb-postgres/) + project_root = os.getcwd() + + # Define paths to the unittest executable and extension repository + # These paths are relative to the project_root + unittest_executable_path = os.path.join(project_root, 'build', 'release', 'test', 'unittest') + extension_repo_path = os.path.join(project_root, 'build', 'release', 'repository') + test_directory = os.path.join(project_root, 'test') + + # --- Pre-flight checks --- + if not os.path.exists(unittest_executable_path): + print(f"Error: Unittest executable not found at '{unittest_executable_path}'. " + "Please ensure the project is built and the path is correct.") + return [] + + if not os.path.exists(extension_repo_path): + print(f"Warning: Extension repository not found at '{extension_repo_path}'. " + "Tests requiring extensions might fail. Please ensure it exists if needed.") + + if not os.path.isdir(test_directory): + print(f"Error: Test directory not found at '{test_directory}'. " + "Please ensure the 'test/' directory exists in the project root.") + return [] + + # Discover all .test files recursively within the 'test/' directory + test_files = glob.glob(os.path.join(test_directory, '**', '*.test'), recursive=True) + + if not test_files: + print(f"No '.test' files found in '{test_directory}'.") + return [] + + print(f"Found {len(test_files)} test files. Running them one by one...\n") + + results = [] + # Prepare the environment variables for the subprocess + # Copy current environment and add/override LOCAL_EXTENSION_REPO + env_vars = os.environ.copy() + env_vars['LOCAL_EXTENSION_REPO'] = extension_repo_path + + test_files = sorted(test_files) + for test_file_full_path in test_files: + print(test_file_full_path) + + sys.exit(1) + + for test_file_full_path in test_files: + # Get the path relative to the project root, as required by the unittest executable + test_name = os.path.relpath(test_file_full_path, project_root) + command = [unittest_executable_path, test_name] + + print(f"--- Running test: {test_name} ---") + start_time = time.monotonic() + process_returncode = -1 # Default to error state + + try: + # Execute the command. capture_output=True captures stdout/stderr. + # text=True decodes stdout/stderr as text. + # check=False prevents an exception for non-zero exit codes, allowing us to record time for failed tests. + process = subprocess.run( + command, + env=env_vars, + capture_output=True, + text=True, + check=False + ) + end_time = time.monotonic() + elapsed_time = end_time - start_time + process_returncode = process.returncode + + results.append({ + 'test_name': test_name, + 'elapsed_time': elapsed_time, + 'returncode': process_returncode + }) + + if process_returncode != 0: + print(f" Test FAILED (Return Code: {process_returncode}) - Elapsed: {elapsed_time:.4f} seconds") + # Uncomment the following lines to see stdout/stderr for failed tests + # print(" --- STDOUT ---") + # print(process.stdout.strip()) + # print(" --- STDERR ---") + # print(process.stderr.strip()) + else: + print(f" Test PASSED - Elapsed: {elapsed_time:.4f} seconds") + + except FileNotFoundError: + print(f" Error: Unittest executable not found or command failed to start for '{test_name}'.") + results.append({'test_name': test_name, 'elapsed_time': -1.0, 'returncode': -1}) + except Exception as e: + print(f" An unexpected error occurred while running '{test_name}': {e}") + results.append({'test_name': test_name, 'elapsed_time': -1.0, 'returncode': -1}) + print("-" * (len(test_name) + 18)) # Separator for clarity + + # Sort results by elapsed time in descending order + # Tests that errored out (elapsed_time = -1.0) will appear at the end due to sorting behavior + sorted_results = sorted(results, key=lambda x: x['elapsed_time'], reverse=True) + + print("\n\n--- Test Summary (Sorted by Elapsed Time, Descending) ---") + if not sorted_results: + print("No test results to display.") + else: + for result in sorted_results: + status = "PASSED" if result['returncode'] == 0 else "FAILED" if result['returncode'] != -1 else "ERROR" + if result['elapsed_time'] >= 0: + print(f"{result['test_name']}: {result['elapsed_time']:.4f} seconds ({status})") + else: + print(f"{result['test_name']}: Error during execution ({status})") + + return sorted_results + +if __name__ == "__main__": + run_and_time_unit_tests() diff --git a/test/sql/scanner/decimals.test b/test/sql/scanner/decimals.test index 3463b89bc..ba94e7d69 100644 --- a/test/sql/scanner/decimals.test +++ b/test/sql/scanner/decimals.test @@ -6,6 +6,8 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE +require-env POSTGRES_TEST_SLOW + statement ok CALL postgres_attach('dbname=postgresscanner'); diff --git a/test/sql/scanner/tpch.test b/test/sql/scanner/tpch.test index 4bcb6e723..2747b3900 100644 --- a/test/sql/scanner/tpch.test +++ b/test/sql/scanner/tpch.test @@ -8,6 +8,8 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE +require-env POSTGRES_TEST_SLOW + statement ok CALL postgres_attach('dbname=postgresscanner', source_schema='tpch', filter_pushdown=true); diff --git a/test/sql/storage/attach_connection_pool.test b/test/sql/storage/attach_connection_pool.test_slow similarity index 96% rename from test/sql/storage/attach_connection_pool.test rename to test/sql/storage/attach_connection_pool.test_slow index 9a7ae9aca..fe3053772 100644 --- a/test/sql/storage/attach_connection_pool.test +++ b/test/sql/storage/attach_connection_pool.test_slow @@ -1,4 +1,4 @@ -# name: test/sql/storage/attach_connection_pool.test +# name: test/sql/storage/attach_connection_pool.test_slow # description: Test that the connection pool correctly limits the number of connections # group: [storage] @@ -6,6 +6,8 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE +require-env POSTGRES_TEST_SLOW + statement ok PRAGMA enable_verification diff --git a/test/sql/storage/attach_large_delete.test b/test/sql/storage/attach_large_delete.test_slow similarity index 88% rename from test/sql/storage/attach_large_delete.test rename to test/sql/storage/attach_large_delete.test_slow index 4f990c3c5..3ce26f970 100644 --- a/test/sql/storage/attach_large_delete.test +++ b/test/sql/storage/attach_large_delete.test_slow @@ -1,4 +1,4 @@ -# name: test/sql/storage/attach_large_delete.test +# name: test/sql/storage/attach_large_delete.test_slow # description: Test deleting many rows from a large table # group: [storage] @@ -6,6 +6,8 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE +require-env POSTGRES_TEST_SLOW + statement ok PRAGMA enable_verification diff --git a/test/sql/storage/attach_multi_join_large.test_slow b/test/sql/storage/attach_multi_join_large.test_slow index 64a6f6626..fdb6cd16e 100644 --- a/test/sql/storage/attach_multi_join_large.test_slow +++ b/test/sql/storage/attach_multi_join_large.test_slow @@ -6,6 +6,8 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE +require-env POSTGRES_TEST_SLOW + statement ok PRAGMA enable_verification diff --git a/test/sql/storage/attach_pages_per_task.test b/test/sql/storage/attach_pages_per_task.test_slow similarity index 88% rename from test/sql/storage/attach_pages_per_task.test rename to test/sql/storage/attach_pages_per_task.test_slow index ef6cf8ce4..c2378df20 100644 --- a/test/sql/storage/attach_pages_per_task.test +++ b/test/sql/storage/attach_pages_per_task.test_slow @@ -1,4 +1,4 @@ -# name: test/sql/storage/attach_pages_per_task.test +# name: test/sql/storage/attach_pages_per_task.test_slow # description: Test the pg_pages_per_task setting # group: [storage] @@ -6,6 +6,8 @@ require postgres_scanner require-env POSTGRES_TEST_DATABASE_AVAILABLE +require-env POSTGRES_TEST_SLOW + statement ok PRAGMA enable_verification