Add Rust (PyO3) native extensions for VectorSets and RediSearch engines

.github/workflows/publish-pypi.yml

@@ -5,39 +5,88 @@ on:
   workflow_dispatch:  # Allow manual triggering
 
 jobs:
-  pypi:
+  # Build wheels for each platform/arch (includes Python code + Rust extension)
+  build-wheels:
+    name: Build wheel (${{ matrix.os }}, ${{ matrix.target }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            target: x86_64
+          - os: ubuntu-latest
+            target: aarch64
+          - os: macos-13
+            target: x86_64
+          - os: macos-14
+            target: aarch64
+          - os: windows-latest
+            target: x64
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build wheels
+      uses: PyO3/maturin-action@v1
+      with:
+        target: ${{ matrix.target }}
+        args: --release --out dist --interpreter 3.10 3.11 3.12 3.13
+        manylinux: auto
+
+    - name: Upload wheels
+      uses: actions/upload-artifact@v4
+      with:
+        name: wheels-${{ matrix.os }}-${{ matrix.target }}
+        path: dist/*.whl
+
+  # Build sdist (fallback for platforms without prebuilt wheels)
+  build-sdist:
+    name: Build sdist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build sdist
+      uses: PyO3/maturin-action@v1
+      with:
+        command: sdist
+        args: --out dist
+
+    - name: Upload sdist
+      uses: actions/upload-artifact@v4
+      with:
+        name: sdist
+        path: dist/*.tar.gz
+
+  # Publish to PyPI
+  publish:
     name: Publish to PyPI
     runs-on: ubuntu-latest
+    needs: [build-wheels, build-sdist]
     environment:
       name: pypi
       url: https://pypi.org/p/vector-benchmark
     permissions:
       contents: read
       id-token: write
     steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-
-    - name: Install Poetry
-      uses: snok/install-poetry@v1
+    - name: Download all artifacts
+      uses: actions/download-artifact@v4
       with:
-        version: latest
-        virtualenvs-create: true
-        virtualenvs-in-project: true
+        path: all-dist
 
-    - name: Build package
-      run: poetry build
-
-    - name: Check package
+    - name: Collect dist files
       run: |
-        pip install twine
-        twine check dist/*
+        mkdir -p dist
+        find all-dist -name '*.whl' -exec cp {} dist/ \;
+        find all-dist -name '*.tar.gz' -exec cp {} dist/ \;
+        echo "Packages to publish:"
+        ls -la dist/
 
     - name: Publish to PyPI
+      uses: PyO3/maturin-action@v1
+      with:
+        command: upload
+        args: --non-interactive --skip-existing dist/*
       env:
-        POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
-      run: poetry publish
+        MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

.github/workflows/test-basic-functionality-redis-vector-sets-rs.yml

@@ -0,0 +1,130 @@
+name: Test Basic Functionality - Redis Vector Sets (Rust)
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test-basic-functionality:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+
+    services:
+      redis:
+        image: redis:8.2-rc1-bookworm
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: latest
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+
+    - name: Install dependencies
+      run: |
+        poetry install --no-root
+        poetry run pip install maturin
+
+    - name: Build Rust extension
+      run: |
+        poetry run maturin develop --release
+        poetry run python -c "import vector_db_benchmark_rs; print('Rust module loaded successfully')"
+
+    - name: Install Redis CLI
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y redis-tools
+
+    - name: Wait for Redis to be ready
+      run: |
+        echo "Waiting for Redis to be ready..."
+        timeout 30 bash -c 'until redis-cli -h localhost -p 6379 ping; do sleep 1; done'
+        echo "Redis is ready!"
+
+    - name: Test describe functionality
+      run: |
+        echo "Testing --describe commands..."
+        poetry run python run.py --describe datasets | head -10
+        poetry run python run.py --describe engines | head -10
+        echo "✅ Describe functionality works"
+
+    - name: Test basic benchmark functionality
+      run: |
+        echo "Testing basic benchmark with Redis Vector Sets (Rust)..."
+        echo "Running: python run.py --host localhost --engines vectorsets-rs-fp32-default --datasets random-100 --queries 10"
+
+        # Run with limited queries for faster testing
+        poetry run python run.py \
+          --host localhost \
+          --engines vectorsets-rs-fp32-default \
+          --datasets random-100 \
+          --queries 10 \
+          --timeout 300
+
+    - name: Verify results were generated
+      run: |
+        echo "Checking if results were generated..."
+        if [ -d "results" ] && [ "$(ls -A results)" ]; then
+          echo "✅ Results directory contains files:"
+          ls -la results/
+
+          # Check for summary file
+          if ls results/*summary.json 1> /dev/null 2>&1; then
+            echo "✅ Summary file found"
+            echo "Sample summary content:"
+            head -20 results/*summary.json
+          else
+            echo "⚠️  No summary file found"
+          fi
+
+          # Check for experiment files
+          if ls results/*vectorsets-rs*.json 1> /dev/null 2>&1; then
+            echo "✅ Experiment result files found"
+          else
+            echo "⚠️  No experiment result files found"
+          fi
+        else
+          echo "❌ No results generated"
+          exit 1
+        fi
+
+    - name: Test with skip options
+      run: |
+        echo "Testing with --skip-upload (search only)..."
+        poetry run python run.py \
+          --host localhost \
+          --engines vectorsets-rs-fp32-default \
+          --datasets random-100 \
+          --queries 50 \
+          --skip-upload \
+          --timeout 180
+
+        echo "✅ Skip upload test completed"
+
+    - name: Cleanup
+      run: |
+        echo "Cleaning up test files..."
+        rm -rf datasets/random-100/
+        rm -rf results/
+        echo "✅ Cleanup completed"

.github/workflows/test-basic-functionality-redisearch-rs.yml

@@ -0,0 +1,130 @@
+name: Test Basic Functionality - Redis RediSearch (Rust)
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test-basic-functionality:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+
+    services:
+      redis:
+        image: redis:8.2-rc1-bookworm
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: latest
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+
+    - name: Install dependencies
+      run: |
+        poetry install --no-root
+        poetry run pip install maturin
+
+    - name: Build Rust extension
+      run: |
+        poetry run maturin develop --release
+        poetry run python -c "import vector_db_benchmark_rs; print('Rust module loaded successfully')"
+
+    - name: Install Redis CLI
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y redis-tools
+
+    - name: Wait for Redis to be ready
+      run: |
+        echo "Waiting for Redis to be ready..."
+        timeout 30 bash -c 'until redis-cli -h localhost -p 6379 ping; do sleep 1; done'
+        echo "Redis is ready!"
+
+    - name: Test describe functionality
+      run: |
+        echo "Testing --describe commands..."
+        poetry run python run.py --describe datasets | head -10
+        poetry run python run.py --describe engines | head -10
+        echo "✅ Describe functionality works"
+
+    - name: Test basic benchmark functionality
+      run: |
+        echo "Testing basic benchmark with Redis (Rust)..."
+        echo "Running: python run.py --host localhost --engines redis-rs-m-16-ef-128 --datasets random-100 --queries 10"
+
+        # Run with limited queries for faster testing
+        poetry run python run.py \
+          --host localhost \
+          --engines redis-rs-m-16-ef-128 \
+          --datasets random-100 \
+          --queries 10 \
+          --timeout 300
+
+    - name: Verify results were generated
+      run: |
+        echo "Checking if results were generated..."
+        if [ -d "results" ] && [ "$(ls -A results)" ]; then
+          echo "✅ Results directory contains files:"
+          ls -la results/
+
+          # Check for summary file
+          if ls results/*summary.json 1> /dev/null 2>&1; then
+            echo "✅ Summary file found"
+            echo "Sample summary content:"
+            head -20 results/*summary.json
+          else
+            echo "⚠️  No summary file found"
+          fi
+
+          # Check for experiment files
+          if ls results/*redis-rs*.json 1> /dev/null 2>&1; then
+            echo "✅ Experiment result files found"
+          else
+            echo "⚠️  No experiment result files found"
+          fi
+        else
+          echo "❌ No results generated"
+          exit 1
+        fi
+
+    - name: Test with skip options
+      run: |
+        echo "Testing with --skip-upload (search only)..."
+        poetry run python run.py \
+          --host localhost \
+          --engines redis-rs-m-16-ef-128 \
+          --datasets random-100 \
+          --queries 50 \
+          --skip-upload \
+          --timeout 180
+
+        echo "✅ Skip upload test completed"
+
+    - name: Cleanup
+      run: |
+        echo "Cleaning up test files..."
+        rm -rf datasets/random-100/
+        rm -rf results/
+        echo "✅ Cleanup completed"