api support for image , spars and rerank

.github/workflows/test.yml

@@ -0,0 +1,167 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - main
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUST_LOG: info
+
+jobs:
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    services:
+      localstack:
+        image: localstack/localstack:4.11.1
+        ports:
+          - 4566:4566
+          - 4571:4571
+        env:
+          SERVICES: s3
+          DEBUG: 1
+          DATA_DIR: /tmp/localstack/data
+        options: >-
+          --health-cmd "awslocal s3 ls || exit 1"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          components: rustfmt, clippy
+
+      - name: Cache cargo registry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-registry-
+
+      - name: Cache cargo index
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-index-${{ hashFiles('Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-index-
+
+      - name: Cache cargo build
+        uses: actions/cache@v3
+        with:
+          path: target
+          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-build-target-
+
+      - name: Cache fastembed models
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/huggingface
+            .fastembed_cache
+          key: ${{ runner.os }}-fastembed-models-v2
+          restore-keys: |
+            ${{ runner.os }}-fastembed-models-
+
+      - name: Run unit tests
+        env:
+          LOCALSTACK_ENDPOINT: http://127.0.0.1:4566
+          FASTEMBED_CACHE_PATH: .fastembed_cache
+        run: cargo test --verbose
+
+  integration-tests:
+    name: Integration Tests - ${{ matrix.test }}
+    runs-on: ubuntu-latest
+    needs: test
+    strategy:
+      matrix:
+        test: [ text, image, sparse, rerank ]
+        include:
+          - test: text
+            timeout: 10
+          - test: image
+            timeout: 15
+          - test: sparse
+            timeout: 10
+          - test: rerank
+            timeout: 10
+    services:
+      localstack:
+        image: localstack/localstack:4.11.1
+        ports:
+          - 4566:4566
+          - 4571:4571
+        env:
+          SERVICES: s3
+          DEBUG: 1
+          DATA_DIR: /tmp/localstack/data
+        options: >-
+          --health-cmd "awslocal s3 ls || exit 1"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          lfs: 'true'
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+
+      - name: Cache cargo registry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-registry-
+
+      - name: Cache cargo index
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-index-${{ hashFiles('Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-index-
+
+      - name: Cache cargo build
+        uses: actions/cache@v3
+        with:
+          path: target
+          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-build-target-
+
+      - name: Cache fastembed models
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/huggingface
+            .fastembed_cache
+          key: ${{ runner.os }}-fastembed-models-v2-${{ matrix.test }}
+          restore-keys: |
+            ${{ runner.os }}-fastembed-models-v2-
+            ${{ runner.os }}-fastembed-models-
+
+      - name: Run ${{ matrix.test }} integration tests
+        env:
+          LOCALSTACK_ENDPOINT: http://127.0.0.1:4566
+          FASTEMBED_CACHE_PATH: .fastembed_cache
+        run: cargo test --features aws --test integration_${{ matrix.test }}_tests -- --nocapture --test-threads=1
+        timeout-minutes: ${{ matrix.timeout }}

Cargo.toml

@@ -17,6 +17,7 @@ fastembed = "5"
 once_cell = "1"
 clap = { version = "4", features = ["derive"] }
 thiserror = "2"
+base64 = "0.22"
 
 # PDF support (optional)
 pdf-extract = { version = "0.10", optional = true }
@@ -59,13 +60,28 @@ strip = true
 tokio-test = "0.4"
 
 [[test]]
-name = "integration_tests"
-path = "tests/integration_tests.rs"
+name = "unit_tests"
+path = "tests/unit_tests.rs"
 required-features = ["aws"]
 
 [[test]]
-name = "unit_tests"
-path = "tests/unit_tests.rs"
+name = "integration_text_tests"
+path = "tests/integration_text_tests.rs"
+required-features = ["aws"]
+
+[[test]]
+name = "integration_image_tests"
+path = "tests/integration_image_tests.rs"
+required-features = ["aws"]
+
+[[test]]
+name = "integration_sparse_tests"
+path = "tests/integration_sparse_tests.rs"
+required-features = ["aws"]
+
+[[test]]
+name = "integration_rerank_tests"
+path = "tests/integration_rerank_tests.rs"
 required-features = ["aws"]
 [build-dependencies]