Add Python SDK integration tests

CLAUDE.md

@@ -8,19 +8,23 @@
 - Run single service: `python run_tests.py --service auth`
 - Run single test: `python -m pytest tests/services/auth/test_api_key_validation.py::TestApiKeyValidation::test_health_check -v`
 - Run by keyword: `python -m pytest tests/services/ -k "test_health_check" -v`
+- Run SDK tests: `python run_tests.py --suite sdk --profile core`
+- Run both suites: `python run_tests.py --suite both --profile core`
+- Run SDK single service: `python run_tests.py --suite sdk --service agents`
 
 ## Environment Variables
 - `VECTARA_API_KEY` — required, Personal API key
 - `VECTARA_BASE_URL` — defaults to `https://api.vectara.io`
 
 ## Project Structure
-- `tests/services/<service>/` — test files organized by API service (auth, corpus, indexing, query, chat, agents)
+- `tests/services/<service>/` — HTTP-level test files organized by API service (auth, corpus, indexing, query, chat, agents)
+- `tests/sdk/<service>/` — SDK-level tests using the `vectara` Python SDK (same service layout)
 - `tests/workflows/` — cross-service end-to-end flow tests
 - `utils/client.py` — Vectara API client (single class, all HTTP methods)
 - `utils/waiters.py` — polling helpers and SSE reader
 - `utils/config.py` — environment-based configuration
 - `fixtures/sample_data.py` — test data
-- `run_tests.py` — CLI runner with `--profile` and `--service` flags
+- `run_tests.py` — CLI runner with `--suite`, `--profile`, and `--service` flags
 
 ## Test Markers
 - Every service test must have exactly one depth marker: `@pytest.mark.sanity`, `@pytest.mark.core`, or `@pytest.mark.regression`
@@ -49,6 +53,8 @@
 - Cleanup resources in `try/finally` blocks.
 - Module-scoped fixtures for shared corpora (read-heavy tests), function-scoped for CRUD tests.
 - **Assertions must verify actual behavior, not just HTTP status.** Always verify response data, field values, and state changes — not just `response.success`.
+- **SDK tests** (`tests/sdk/`) use `sdk_client` and `sdk_shared_agent` fixtures from `tests/sdk/conftest.py`. Tests that mutate shared fixtures must be marked `@pytest.mark.serial`.
+- SDK tests require `vectara>=0.4.3`. Use `--suite sdk` or `--suite both` to include them.
 
 ## General Behavior
 - Treat the user as an expert.

README.md

@@ -67,6 +67,29 @@ python run_tests.py --profile core --html-report --json-report  # Both
 
 Reports are saved to `reports/` with descriptive names like `test_report_20260403_core.html`.
 
+### SDK Tests
+
+The test suite includes SDK-level tests that exercise the `vectara` Python SDK (`vectara>=0.4.3`).
+
+```bash
+# Install dependencies (includes the vectara SDK)
+pip install -r requirements.txt
+
+# Run SDK tests only
+python run_tests.py --suite sdk --profile core
+
+# Run both HTTP and SDK test suites
+python run_tests.py --suite both --profile core
+
+# Run SDK tests for a specific service
+python run_tests.py --suite sdk --service agents
+
+# Run SDK tests against a staging environment
+VECTARA_BASE_URL=https://staging.vectara.io python run_tests.py --suite sdk --profile core
+```
+
+The `--suite` flag accepts `http` (default, backward compatible), `sdk`, or `both`.
+
 ### Parallel Execution
 
 ```bash
@@ -89,6 +112,9 @@ python run_tests.py --profile core -p 4
 ```
 tests/
 ├── conftest.py                  # Marker registration, shared fixtures
+├── sdk/
+│   ├── conftest.py              # SDK client + shared agent fixtures
+│   └── agents/                  # SDK agent tests (config, identity, compaction, sessions)
 ├── services/
 │   ├── conftest.py              # Shared corpus/agent fixtures
 │   ├── agents/                  # Agent CRUD, execution, sessions, compaction, context, corpora search

requirements.in

@@ -26,3 +26,7 @@ python-dateutil>=2.8.2
 
 # JSON schema validation
 jsonschema>=4.21.0
+
+# Vectara Python SDK (for SDK integration tests)
+vectara>=0.4.3
+httpx-sse>=0.4.0

run_tests.py

@@ -19,6 +19,12 @@
     # Run with a depth profile
     python run_tests.py --profile core
 
+    # Run SDK tests
+    python run_tests.py --suite sdk --profile core
+
+    # Run both HTTP and SDK tests
+    python run_tests.py --suite both --profile core
+
     # Generate HTML report
     python run_tests.py --html-report
 """
@@ -48,7 +54,7 @@
     "full": None,  # no marker filter
 }
 
-# Available services (auto-discovered from tests/services/ subdirectories)
+# Available services (auto-discovered from tests/services/ and tests/sdk/ subdirectories)
 AVAILABLE_SERVICES = ["agents", "auth", "chat", "corpus", "indexing", "llm", "pipelines", "query", "tools", "users"]
 
 
@@ -122,13 +128,24 @@ def build_pytest_args(args, services, profile):
     # --- marker expression from profile ---
     marker_expr = PROFILE_MARKERS.get(profile)
 
-    # --- target directories ---
+    # --- target directories based on suite ---
+    suite = args.suite
     if services:
-        targets = [f"tests/services/{svc}/" for svc in services]
+        if suite == "http":
+            targets = [f"tests/services/{svc}/" for svc in services]
+        elif suite == "sdk":
+            targets = [f"tests/sdk/{svc}/" for svc in services]
+        else:  # both
+            targets = [f"tests/services/{svc}/" for svc in services] + [f"tests/sdk/{svc}/" for svc in services]
     elif profile == "full":
         targets = ["tests/"]
     else:
-        targets = ["tests/services/"]
+        if suite == "http":
+            targets = ["tests/services/"]
+        elif suite == "sdk":
+            targets = ["tests/sdk/"]
+        else:  # both
+            targets = ["tests/services/", "tests/sdk/"]
 
     # Build a descriptive label for report filenames
     if services:
@@ -247,6 +264,9 @@ def main():
   python run_tests.py --html-report                            # Generate HTML report
   python run_tests.py --llm-name mockingbird-2.0               # Specify LLM model
   python run_tests.py --generation-preset vectara-summary-ext-24-05-med-omni
+  python run_tests.py --suite sdk --profile core               # Run SDK tests only
+  python run_tests.py --suite both --service agents            # Run HTTP + SDK agent tests
+  python run_tests.py --suite both --profile core              # Run both suites, core profile
 
 Environment Variables:
   VECTARA_API_KEY            Your Personal API key (recommended for CI/CD)
@@ -278,6 +298,14 @@ def main():
         help="Generation preset name (or set VECTARA_GENERATION_PRESET env var)",
     )
 
+    # Suite selection
+    parser.add_argument(
+        "--suite",
+        choices=["http", "sdk", "both"],
+        default="http",
+        help="Test suite to run: http (default), sdk, or both",
+    )
+
     # Profile and service selection
     parser.add_argument(
         "--profile",
@@ -372,6 +400,7 @@ def main():
         table.add_column("Setting", style="cyan")
         table.add_column("Value")
 
+        table.add_row("Suite", f"[bold]{args.suite}[/bold]")
         table.add_row("Profile", f"[bold]{profile}[/bold]")
 
         if services:

tests/conftest.py

@@ -106,8 +106,8 @@ def pytest_collection_modifyitems(config, items):
         if "/workflows/" in str(item.fspath):
             continue
 
-        # Only enforce on service tests (under tests/services/).
-        if "/services/" not in str(item.fspath):
+        # Only enforce on service and SDK tests.
+        if "/services/" not in str(item.fspath) and "/sdk/" not in str(item.fspath):
             continue
 
         marker_names = {m.name for m in item.iter_markers()}