From 097d018a3810527d3623b65e4fd60f195ea59120 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Tue, 24 Feb 2026 19:14:44 -0800 Subject: [PATCH 1/3] Commit --- .github/workflows/e2e.yml | 71 +++++++++++++++++++++++++++++++++ docs/environment-variables.md | 9 ++++- docs/rust-core-bindings.md | 36 ++++++++--------- drift/core/drift_sdk.py | 29 ++++++++++++++ drift/core/rust_core_binding.py | 34 +++++++++++++++- pyproject.toml | 2 +- uv.lock | 17 ++++---- 7 files changed, 168 insertions(+), 30 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b918792..d9e0e1c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -196,3 +196,74 @@ jobs: docker volume prune -f || true # Clean up networks docker network prune -f || true + + non-rust-smoke: + name: E2E Non-Rust Smoke - requests + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Setup Python + run: uv python install 3.9 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker + + - name: Install SDK dependencies + run: uv sync --all-extras + + - name: Build SDK + run: uv build + + - name: Verify SDK build + run: | + ls -la dist/ || (echo "dist folder not found!" && exit 1) + test -f dist/*.whl || (echo "SDK build incomplete!" && exit 1) + + - name: Get latest Tusk CLI version + id: tusk-version + run: | + VERSION=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/Use-Tusk/tusk-drift-cli/releases/latest" \ + | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Latest Tusk CLI version: $VERSION" + + - name: Build base image + env: + DOCKER_DEFAULT_PLATFORM: linux/amd64 + run: | + docker build \ + --build-arg TUSK_CLI_VERSION=${{ steps.tusk-version.outputs.version }} \ + -t python-e2e-base:latest \ + -f drift/instrumentation/e2e_common/Dockerfile.base \ + . + + - name: Run non-rust smoke test + env: + DOCKER_DEFAULT_PLATFORM: linux/amd64 + TUSK_CLI_VERSION: ${{ steps.tusk-version.outputs.version }} + TUSK_USE_RUST_CORE: "0" + run: | + chmod +x ./drift/instrumentation/requests/e2e-tests/run.sh + cd ./drift/instrumentation/requests/e2e-tests && ./run.sh 8000 + + - name: Cleanup Docker resources + if: always() + run: | + # Stop all running containers + docker ps -aq | xargs -r docker stop || true + docker ps -aq | xargs -r docker rm || true + # Clean up volumes + docker volume prune -f || true + # Clean up networks + docker network prune -f || true diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 6cc076a..a535a66 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -136,13 +136,15 @@ These variables control optional Rust-accelerated paths in the SDK. | Variable | Description | Default | | --- | --- | --- | -| `TUSK_USE_RUST_CORE` | Enables Rust binding usage when available (`1`, `true`, `yes`) | `0` (disabled) | +| `TUSK_USE_RUST_CORE` | Controls Rust binding usage. Truthy (`1`, `true`, `yes`, `on`) enables, falsy (`0`, `false`, `no`, `off`) disables. | Enabled when unset | | `TUSK_SKIP_PROTO_VALIDATION` | Skips expensive protobuf validation in hot path (`1`, `true`, `yes`) | `0` (disabled) | **Notes:** - The SDK is fail-open: if Rust bindings are unavailable or a Rust call fails, it falls back to Python implementation. +- `TUSK_USE_RUST_CORE` defaults to enabled when unset. - `TUSK_USE_RUST_CORE` does not install Rust bindings automatically. The `drift-core-python` package still must be installed in your environment. +- If Rust is enabled but bindings cannot be loaded, the SDK logs startup fallback and continues on Python paths. - `TUSK_SKIP_PROTO_VALIDATION` is performance-focused and should be used with confidence in parity tests and serialization correctness. See [`rust-core-bindings.md`](./rust-core-bindings.md) for more details. @@ -150,9 +152,12 @@ See [`rust-core-bindings.md`](./rust-core-bindings.md) for more details. **Example usage:** ```bash -# Enable Rust path (if drift-core-python is installed) +# Explicitly enable Rust path (also the default when unset) TUSK_USE_RUST_CORE=1 python app.py +# Explicitly disable Rust path +TUSK_USE_RUST_CORE=0 python app.py + # Enable Rust path and skip proto validation TUSK_USE_RUST_CORE=1 TUSK_SKIP_PROTO_VALIDATION=1 python app.py ``` diff --git a/docs/rust-core-bindings.md b/docs/rust-core-bindings.md index d4a691c..018de13 100644 --- a/docs/rust-core-bindings.md +++ b/docs/rust-core-bindings.md @@ -14,13 +14,22 @@ At a high level: ## Enablement -Set: +Rust is enabled by default when `TUSK_USE_RUST_CORE` is unset. + +Use `TUSK_USE_RUST_CORE` to explicitly override behavior: + +- Truthy: `1`, `true`, `yes`, `on` +- Falsy: `0`, `false`, `no`, `off` + +Examples: ```bash +# Explicitly enable (same as unset) TUSK_USE_RUST_CORE=1 -``` -Truthy values are `1`, `true`, and `yes` (case-insensitive). Any other value is treated as disabled. +# Explicitly disable +TUSK_USE_RUST_CORE=0 +``` ## Installation Requirements @@ -37,23 +46,13 @@ You can install the SDK with Rust bindings via extras: pip install "tusk-drift-python-sdk[rust]" ``` -## Wheel Platform Coverage - -Based on the current `drift-core` publish workflow, prebuilt wheels are built for: - -- Linux `x86_64-unknown-linux-gnu` -- Linux `aarch64-unknown-linux-gnu` -- macOS Apple Silicon `aarch64-apple-darwin` -- Windows `x86_64-pc-windows-msvc` +## Platform Compatibility -Likely missing prebuilt wheels (source build fallback required) include: +`drift-core` publishes native artifacts across a defined support matrix. See: -- macOS Intel (`x86_64-apple-darwin`) -- Linux musl targets (e.g. Alpine) -- Windows ARM64 -- Other uncommon Python/platform combinations not covered by release artifacts +- [`drift-core` compatibility matrix](https://github.com/Use-Tusk/drift-core/blob/main/docs/compatibility-matrix.md) -If no wheel matches the environment, `pip` may attempt a source build of `drift-core-python`, which typically requires a Rust toolchain and native build prerequisites. +If no compatible wheel exists for your environment, `pip` may attempt a source build of `drift-core-python`, which typically requires a Rust toolchain and native build prerequisites. ## Fallback Behavior @@ -62,6 +61,7 @@ The bridge module is fail-open: - Rust calls are guarded. - On import failures or call exceptions, the corresponding helper returns `None`. - Calling code then uses the existing Python implementation. +- On startup, the SDK logs whether Rust is enabled/disabled and whether it had to fall back to Python. This means users do not need Rust installed to run the SDK when Rust acceleration is disabled or unavailable. @@ -80,7 +80,7 @@ Use with care: ## Practical Guidance -- Default production-safe posture: leave Rust disabled unless you have tested your deployment matrix. +- Default production-safe posture: keep Rust enabled (default) only on tested deployment matrices. - Performance posture: enable Rust + benchmark on your workloads before broad rollout. - Reliability posture: keep parity tests and smoke tests in CI to detect drift between Python and Rust paths. diff --git a/drift/core/drift_sdk.py b/drift/core/drift_sdk.py index 631de77..82580fd 100644 --- a/drift/core/drift_sdk.py +++ b/drift/core/drift_sdk.py @@ -78,6 +78,34 @@ def _generate_sdk_instance_id(self) -> str: random_suffix = "".join(random.choices("abcdefghijklmnopqrstuvwxyz0123456789", k=9)) return f"sdk-{timestamp_ms}-{random_suffix}" + @staticmethod + def _log_rust_core_startup_status() -> None: + from .rust_core_binding import get_rust_core_startup_status + + status = get_rust_core_startup_status() + env_display = status["raw_env"] if status["raw_env"] is not None else "" + + if status["reason"] == "invalid_env_value_defaulted": + logger.warning( + "Invalid TUSK_USE_RUST_CORE value '%s'; defaulting to enabled rust core path.", + env_display, + ) + + if not status["enabled"]: + logger.info("Rust core path disabled at startup (env=%s, reason=%s).", env_display, status["reason"]) + return + + if status["binding_loaded"]: + logger.info("Rust core path enabled at startup (env=%s, reason=%s).", env_display, status["reason"]) + return + + logger.warning( + "Rust core path requested but binding unavailable; falling back to Python path (env=%s, reason=%s, error=%s).", + env_display, + status["reason"], + status["binding_error"], + ) + @classmethod def initialize( cls, @@ -156,6 +184,7 @@ def initialize( logger.debug("SDK disabled via environment variable") return instance + instance._log_rust_core_startup_status() logger.debug(f"Initializing in {instance.mode} mode") effective_transforms = transforms diff --git a/drift/core/rust_core_binding.py b/drift/core/rust_core_binding.py index ff139f6..b885d97 100644 --- a/drift/core/rust_core_binding.py +++ b/drift/core/rust_core_binding.py @@ -15,14 +15,30 @@ _binding_module = None _binding_load_attempted = False +_binding_load_error: str | None = None + +_RUST_TRUTHY = {"1", "true", "yes", "on"} +_RUST_FALSY = {"0", "false", "no", "off"} + +def _rust_env_decision() -> tuple[bool, str, str | None]: + raw = os.getenv("TUSK_USE_RUST_CORE") + if raw is None: + return True, "default_on", None + normalized = raw.strip().lower() + if normalized in _RUST_TRUTHY: + return True, "env_enabled", raw + if normalized in _RUST_FALSY: + return False, "env_disabled", raw + return True, "invalid_env_value_defaulted", raw def _enabled() -> bool: - return os.getenv("TUSK_USE_RUST_CORE", "0").lower() in {"1", "true", "yes"} + enabled, _, _ = _rust_env_decision() + return enabled def _load_binding(): - global _binding_module, _binding_load_attempted + global _binding_module, _binding_load_attempted, _binding_load_error if _binding_load_attempted: return _binding_module _binding_load_attempted = True @@ -30,12 +46,26 @@ def _load_binding(): import drift_core as binding _binding_module = binding + _binding_load_error = None except Exception as exc: # pragma: no cover - depends on runtime env logger.debug("Rust core binding not available: %s", exc) _binding_module = None + _binding_load_error = f"{type(exc).__name__}: {exc}" return _binding_module +def get_rust_core_startup_status() -> dict[str, Any]: + enabled, reason, raw_env = _rust_env_decision() + loaded = _load_binding() if enabled else None + return { + "enabled": enabled, + "reason": reason, + "raw_env": raw_env, + "binding_loaded": loaded is not None, + "binding_error": _binding_load_error, + } + + def normalize_and_hash_jsonable(value: Any) -> tuple[Any, str] | None: """Return normalized JSON value and deterministic hash via Rust binding.""" if not _enabled(): diff --git a/pyproject.toml b/pyproject.toml index cc31a61..f7f6c60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ flask = ["Flask>=3.1.2"] fastapi = ["fastapi>=0.115.6", "uvicorn>=0.34.2", "starlette<0.42.0"] django = ["Django>=4.2"] -rust = ["drift-core-python>=0.1.6"] +rust = ["drift-core-python>=0.1.7"] dev = [ "Flask>=3.1.2", "fastapi>=0.115.6", diff --git a/uv.lock b/uv.lock index d7f6770..323c5a0 100644 --- a/uv.lock +++ b/uv.lock @@ -813,14 +813,17 @@ wheels = [ [[package]] name = "drift-core-python" -version = "0.1.6" +version = "0.1.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/51/fcd6ed8476533cff9f0d28836ef72fb716da4558331f1d440c67fa0e7dc9/drift_core_python-0.1.6.tar.gz", hash = "sha256:2cb1227a36608cdbfa9bcf239a5b4de6eda9473973ad9b49872b0025814b7a62", size = 12276 } +sdist = { url = "https://files.pythonhosted.org/packages/19/89/57ab05ec81807ab2f646f35a30604fc6115f709d5849c7e63ac931fd1eb8/drift_core_python-0.1.7.tar.gz", hash = "sha256:d2a93142ffc81939fd7d41f6b2c3ab1cdaf6db985ac5cb5235c2a090af2d7793", size = 13510 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/49/667e83c15ff5c45a7f6c2636f4111768b7127b7a2c7c06cc348f54eeab81/drift_core_python-0.1.6-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:072e7c63a72aa0fbe274c1e31dab5eca2473507b30a82de6dab7202bb89f3407", size = 365638 }, - { url = "https://files.pythonhosted.org/packages/8b/16/95e0cbeab23da89a88d85ab251e7cce62002f628541f32ff595bcef11113/drift_core_python-0.1.6-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f2c7c7dff69566b3a0baef276988181c55673bef10519adaf77f81e0b9cda42", size = 404958 }, - { url = "https://files.pythonhosted.org/packages/cc/51/69e01bd3c120aed372e5646969d2cbfa923d8e458e6f0145accd03aed216/drift_core_python-0.1.6-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9669dc49925be11e8cbcd93bb31d4006adb952ae08a581d1e00b1746829944e", size = 425428 }, - { url = "https://files.pythonhosted.org/packages/c1/61/5156fca92a2a91eaa4c4adf46649a8b40b71f12fbba29a7a83ceece16604/drift_core_python-0.1.6-cp39-abi3-win_amd64.whl", hash = "sha256:774b6ae381433cd3683cb990d502b5fae68c49fb2a0895ec1b96b68177ca6e2a", size = 256566 }, + { url = "https://files.pythonhosted.org/packages/fb/61/65899e3f7a978bbe0438181bbd932c41614498b2164f443586568d18586e/drift_core_python-0.1.7-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f68e221c810974ab4125e97df1b04b6c98049835fd82a9f67e9f78d9facc9275", size = 379149 }, + { url = "https://files.pythonhosted.org/packages/92/56/01f394c72dc4d8a2b227520e56685bc3139fb1b5fb9eec70143530215066/drift_core_python-0.1.7-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8c81c0a7b4b478e9527de9e1b92ee7c33ebd1e5e927da1504266c445311a3fba", size = 365822 }, + { url = "https://files.pythonhosted.org/packages/21/65/c4406a8c78dfbe4a3d75a8569fb3abcd7027ffb54ae405911b538cc5015c/drift_core_python-0.1.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89faea31af9446cbee1dea20d18dac51dc377ca9a0e92f258bd3b2f0798db699", size = 404792 }, + { url = "https://files.pythonhosted.org/packages/19/43/bcf487715e872807b5b52dbec41309a248eb9917dd0d321a34e877af90e7/drift_core_python-0.1.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9beae827e3bbe09044e316f08b511dae5c76578c67392f6b476bea950f5e02b1", size = 425982 }, + { url = "https://files.pythonhosted.org/packages/2a/77/11f80a638fc2af26ee8b60779ac29e5b10ff2e8984aac64fd0e984a1c976/drift_core_python-0.1.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b4acdbacb99ef03c7aa6d4e94e51793c7423985566d9741305699adcad99564d", size = 582006 }, + { url = "https://files.pythonhosted.org/packages/b3/8a/2804b5027cdff9c9c6d845595a798596410016086ea0e8e2997f06e4b1e0/drift_core_python-0.1.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a40d2d87a86ff9fad8df48358badb6a8f15a4717b377a3915bc5db5439f4f17", size = 626965 }, + { url = "https://files.pythonhosted.org/packages/8a/0e/08ed5090239e1bf22693b6630b794828c5997899830e1762ed94a0d31816/drift_core_python-0.1.7-cp39-abi3-win_amd64.whl", hash = "sha256:6ac4b100bb7cfdcd3931b87ccca987e44ffa4b8798e14ccbc65c8c3a39492749", size = 256904 }, ] [[package]] @@ -2405,7 +2408,7 @@ requires-dist = [ { name = "aiofiles", specifier = ">=23.0.0" }, { name = "aiohttp", specifier = ">=3.9.0" }, { name = "django", marker = "extra == 'django'", specifier = ">=4.2" }, - { name = "drift-core-python", marker = "extra == 'rust'", specifier = ">=0.1.6" }, + { name = "drift-core-python", marker = "extra == 'rust'", specifier = ">=0.1.7" }, { name = "fastapi", marker = "extra == 'dev'", specifier = ">=0.115.6" }, { name = "fastapi", marker = "extra == 'fastapi'", specifier = ">=0.115.6" }, { name = "flask", marker = "extra == 'dev'", specifier = ">=3.1.2" }, From 0a51ad79eecd8f2baaf5cd63a9774bbf6ba6adfc Mon Sep 17 00:00:00 2001 From: JY Tan Date: Tue, 24 Feb 2026 19:18:40 -0800 Subject: [PATCH 2/3] Lint --- drift/core/rust_core_binding.py | 1 + 1 file changed, 1 insertion(+) diff --git a/drift/core/rust_core_binding.py b/drift/core/rust_core_binding.py index b885d97..b9b298a 100644 --- a/drift/core/rust_core_binding.py +++ b/drift/core/rust_core_binding.py @@ -20,6 +20,7 @@ _RUST_TRUTHY = {"1", "true", "yes", "on"} _RUST_FALSY = {"0", "false", "no", "off"} + def _rust_env_decision() -> tuple[bool, str, str | None]: raw = os.getenv("TUSK_USE_RUST_CORE") if raw is None: From 7722a375d0c365dafda2f89e21d8640e9d4b590e Mon Sep 17 00:00:00 2001 From: JY Tan Date: Tue, 24 Feb 2026 20:08:38 -0800 Subject: [PATCH 3/3] Update README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index a59491b..2ccc403 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,15 @@ Alternatively, you can set up Tusk Drift manually: 1. Install the SDK: ```bash + # Use Rust bindings for better performance + pip install tusk-drift-python-sdk[rust] + + # Fallback if no platform-compatible wheel pip install tusk-drift-python-sdk ``` + *For more information about Rust acceleration, refer to [this doc](docs/rust-core-bindings).* + 2. Create configuration: Run `tusk init` to create your `.tusk/config.yaml` config file interactively, or create it manually per the [configuration docs](https://github.com/Use-Tusk/tusk-drift-cli/blob/main/docs/configuration.md). 3. Initialize the SDK: Refer to the [initialization guide](docs/initialization.md) to instrument the SDK in your service.