From 545a9a20a54d81b2a6c03dba208c68bb0725a165 Mon Sep 17 00:00:00 2001 From: Jilles van Gurp Date: Thu, 2 Apr 2026 09:31:07 +0200 Subject: [PATCH 1/4] Add replay connector and shared hub scripts --- README.md | 6 + connectors/README.md | 1 + connectors/gtfs/README.md | 17 +- .../gtfs/scripts/check_geofence_alignment.py | 195 -------------- connectors/opensky/README.md | 14 +- .../opensky/scripts/log_collision_events.py | 26 -- .../opensky/scripts/log_fence_events.py | 26 -- connectors/opensky/scripts/log_locations.py | 26 -- .../opensky/scripts/ws_ndjson_logger.py | 98 ------- connectors/replay/.env.example | 9 + connectors/replay/README.md | 173 +++++++++++++ connectors/replay/connector.py | 177 +++++++++++++ connectors/replay/hub_client.py | 166 ++++++++++++ connectors/replay/pyproject.toml | 12 + connectors/replay/replay_support.py | 242 ++++++++++++++++++ connectors/replay/uv.lock | 142 ++++++++++ docs/index.md | 6 +- .../check_fence_alignment.py | 29 ++- .../log_collision_events.py | 17 +- .../scripts => scripts}/log_fence_events.py | 17 +- .../gtfs/scripts => scripts}/log_locations.py | 17 +- scripts/pyproject.toml | 12 + scripts/uv.lock | 142 ++++++++++ .../scripts => scripts}/ws_ndjson_logger.py | 40 ++- 24 files changed, 1178 insertions(+), 432 deletions(-) delete mode 100644 connectors/gtfs/scripts/check_geofence_alignment.py delete mode 100644 connectors/opensky/scripts/log_collision_events.py delete mode 100644 connectors/opensky/scripts/log_fence_events.py delete mode 100644 connectors/opensky/scripts/log_locations.py delete mode 100644 connectors/opensky/scripts/ws_ndjson_logger.py create mode 100644 connectors/replay/.env.example create mode 100644 connectors/replay/README.md create mode 100644 connectors/replay/connector.py create mode 100644 connectors/replay/hub_client.py create mode 100644 connectors/replay/pyproject.toml create mode 100644 connectors/replay/replay_support.py create mode 100644 connectors/replay/uv.lock rename {connectors/opensky/scripts => scripts}/check_fence_alignment.py (91%) rename {connectors/gtfs/scripts => scripts}/log_collision_events.py (55%) rename {connectors/gtfs/scripts => scripts}/log_fence_events.py (55%) rename {connectors/gtfs/scripts => scripts}/log_locations.py (55%) create mode 100644 scripts/pyproject.toml create mode 100644 scripts/uv.lock rename {connectors/gtfs/scripts => scripts}/ws_ndjson_logger.py (71%) diff --git a/README.md b/README.md index bf84c1c..d6f1477 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,12 @@ Notes: - [connectors/gtfs/README.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/gtfs/README.md) - [connectors/opensky/README.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/opensky/README.md) +## Utility Scripts +- [scripts/log_locations.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_locations.py) +- [scripts/log_fence_events.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_fence_events.py) +- [scripts/log_collision_events.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_collision_events.py) +- [scripts/check_fence_alignment.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/check_fence_alignment.py) + ## Engineering Docs - [engineering/index.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/engineering/index.md) - [engineering/testing.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/engineering/testing.md) diff --git a/connectors/README.md b/connectors/README.md index b1c941d..87bf4d3 100644 --- a/connectors/README.md +++ b/connectors/README.md @@ -17,3 +17,4 @@ Available connector demos: - [`connectors/local-hub`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/local-hub): reusable local hub, Postgres, Dex, and Mosquitto stack - [`connectors/gtfs`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/gtfs): GTFS-RT vehicle updates and station fence bootstrap - [`connectors/opensky`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/opensky): OpenSky aircraft positions with airport-sector fences +- [`connectors/replay`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/replay): diagnostic NDJSON trace replay with timestamp correction, acceleration, and interpolation diff --git a/connectors/gtfs/README.md b/connectors/gtfs/README.md index 9fb8d30..3600fdb 100644 --- a/connectors/gtfs/README.md +++ b/connectors/gtfs/README.md @@ -18,11 +18,6 @@ The checked-in defaults target the live Grand Dole network: `location_updates` over WebSocket - `station_polygons.py`: creates one station `Zone` and one station `Fence` per station in the hub -- `scripts/log_locations.py`: subscribes to `location_updates` and writes NDJSON -- `scripts/log_fence_events.py`: subscribes to `fence_events` and writes NDJSON -- `scripts/log_collision_events.py`: subscribes to `collision_events` and writes NDJSON -- `scripts/check_geofence_alignment.py`: compares logged locations against the - current station fences - `hub_client.py`: shared REST and WebSocket helper code - `gtfs_support.py`: GTFS parsing, GTFS-RT decoding, and station polygon generation helpers @@ -103,12 +98,12 @@ uv run --project connectors/gtfs python connectors/gtfs/station_polygons.py --en uv run --project connectors/gtfs python connectors/gtfs/connector.py --env-file connectors/gtfs/.env.local ``` -7. Optional: record live WebSocket topics to NDJSON: +7. Optional: record live WebSocket topics to NDJSON with the shared root scripts: ```bash -uv run --project connectors/gtfs python connectors/gtfs/scripts/log_locations.py --env-file connectors/gtfs/.env.local -uv run --project connectors/gtfs python connectors/gtfs/scripts/log_fence_events.py --env-file connectors/gtfs/.env.local -uv run --project connectors/gtfs python connectors/gtfs/scripts/log_collision_events.py --env-file connectors/gtfs/.env.local +uv run --project scripts python scripts/log_locations.py --env-file connectors/gtfs/.env.local --output connectors/gtfs/logs/location_updates.ndjson +uv run --project scripts python scripts/log_fence_events.py --env-file connectors/gtfs/.env.local --output connectors/gtfs/logs/fence_events.ndjson +uv run --project scripts python scripts/log_collision_events.py --env-file connectors/gtfs/.env.local --output connectors/gtfs/logs/collision_events.ndjson ``` For a single GTFS-RT fetch during local testing: @@ -188,10 +183,10 @@ can emit `fence_events` when vehicle trackables enter or leave those station polygons. Subscribe to `fence_events` or `fence_events:geojson` to observe arrival and departure behavior. -The `scripts/` directory also includes a simple alignment checker: +The shared root `scripts/` directory also includes a simple alignment checker: ```bash -uv run --project connectors/gtfs python connectors/gtfs/scripts/check_geofence_alignment.py --env-file connectors/gtfs/.env.local +uv run --project scripts python scripts/check_fence_alignment.py --env-file connectors/gtfs/.env.local --locations-log connectors/gtfs/logs/location_updates.ndjson ``` ## Limitations diff --git a/connectors/gtfs/scripts/check_geofence_alignment.py b/connectors/gtfs/scripts/check_geofence_alignment.py deleted file mode 100644 index e62f172..0000000 --- a/connectors/gtfs/scripts/check_geofence_alignment.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -"""Compare logged locations against current hub fences and summarize proximity.""" - -from __future__ import annotations - -import argparse -import json -import math -import os -import sys -from pathlib import Path -from typing import Any - -import requests - -SCRIPT_DIR = Path(__file__).resolve().parent -ROOT_DIR = SCRIPT_DIR.parent -sys.path.insert(0, str(ROOT_DIR)) - -from gtfs_support import load_env_file # noqa: E402 - - -def build_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--env-file", default=os.getenv("GTFS_ENV_FILE")) - parser.add_argument("--locations-log", default="connectors/gtfs/logs/location_updates.ndjson") - parser.add_argument("--http-url", default=os.getenv("HUB_HTTP_URL")) - parser.add_argument("--token", default=os.getenv("HUB_TOKEN")) - return parser - - -def main() -> int: - args = build_argument_parser().parse_args() - load_env_file(args.env_file) - - http_url = args.http_url or os.getenv("HUB_HTTP_URL") - if not http_url: - raise SystemExit("HUB_HTTP_URL or --http-url is required") - - fences = fetch_fences(http_url, args.token or os.getenv("HUB_TOKEN")) - fence_polygons = [normalize_fence(fence) for fence in fences if normalize_fence(fence) is not None] - locations = load_locations(Path(args.locations_log)) - - summary = summarize_proximity(locations, fence_polygons) - print(json.dumps(summary, indent=2)) - return 0 - - -def fetch_fences(http_url: str, token: str | None) -> list[dict[str, Any]]: - headers = {"Accept": "application/json"} - if token: - headers["Authorization"] = f"Bearer {token}" - response = requests.get(f"{http_url.rstrip('/')}/v2/fences", headers=headers, timeout=30) - response.raise_for_status() - return response.json() - - -def normalize_fence(fence: dict[str, Any]) -> dict[str, Any] | None: - region = fence.get("region") or {} - if region.get("type") != "Polygon": - return None - coordinates = region.get("coordinates") or [] - if not coordinates or not coordinates[0]: - return None - return { - "id": fence.get("id"), - "foreign_id": fence.get("foreign_id"), - "name": fence.get("name"), - "ring": coordinates[0], - } - - -def load_locations(path: Path) -> list[dict[str, Any]]: - results: list[dict[str, Any]] = [] - if not path.exists(): - return results - with path.open("r", encoding="utf-8") as handle: - for line in handle: - if not line.strip(): - continue - record = json.loads(line) - message = record.get("message", {}) - if message.get("event") != "message": - continue - for payload in message.get("payload", []): - position = payload.get("position") or {} - coordinates = position.get("coordinates") or [] - if len(coordinates) < 2: - continue - results.append( - { - "received_at": record.get("received_at"), - "provider_id": payload.get("provider_id"), - "trackables": payload.get("trackables", []), - "longitude": coordinates[0], - "latitude": coordinates[1], - } - ) - return results - - -def summarize_proximity( - locations: list[dict[str, Any]], - fences: list[dict[str, Any]], -) -> dict[str, Any]: - inside_count = 0 - min_distance = None - closest_examples: list[dict[str, Any]] = [] - - for location in locations: - best = None - point = (location["longitude"], location["latitude"]) - for fence in fences: - if point_in_polygon(point, fence["ring"]): - inside_count += 1 - best = {"distance_meters": 0.0, "fence": fence} - break - distance = polygon_distance_meters(point, fence["ring"]) - if best is None or distance < best["distance_meters"]: - best = {"distance_meters": distance, "fence": fence} - - if best is None: - continue - distance = best["distance_meters"] - if min_distance is None or distance < min_distance: - min_distance = distance - closest_examples.append( - { - "received_at": location["received_at"], - "provider_id": location["provider_id"], - "trackables": location["trackables"], - "distance_meters": round(distance, 2), - "fence_id": best["fence"]["id"], - "fence_name": best["fence"]["name"], - "fence_foreign_id": best["fence"]["foreign_id"], - } - ) - - closest_examples.sort(key=lambda item: item["distance_meters"]) - return { - "location_count": len(locations), - "fence_count": len(fences), - "locations_inside_any_fence": inside_count, - "closest_distance_meters": None if min_distance is None else round(min_distance, 2), - "closest_examples": closest_examples[:10], - } - - -def point_in_polygon(point: tuple[float, float], ring: list[list[float]]) -> bool: - x, y = point - inside = False - for index in range(len(ring) - 1): - x1, y1 = ring[index] - x2, y2 = ring[index + 1] - intersects = ((y1 > y) != (y2 > y)) and ( - x < (x2 - x1) * (y - y1) / ((y2 - y1) or 1e-12) + x1 - ) - if intersects: - inside = not inside - return inside - - -def polygon_distance_meters(point: tuple[float, float], ring: list[list[float]]) -> float: - best = math.inf - for index in range(len(ring) - 1): - best = min(best, segment_distance_meters(point, tuple(ring[index]), tuple(ring[index + 1]))) - return best - - -def segment_distance_meters( - point: tuple[float, float], - start: tuple[float, float], - end: tuple[float, float], -) -> float: - lon_scale = 111_320.0 * math.cos(math.radians(point[1])) - lat_scale = 111_320.0 - - px, py = point[0] * lon_scale, point[1] * lat_scale - sx, sy = start[0] * lon_scale, start[1] * lat_scale - ex, ey = end[0] * lon_scale, end[1] * lat_scale - - dx = ex - sx - dy = ey - sy - if dx == 0 and dy == 0: - return math.hypot(px - sx, py - sy) - - t = ((px - sx) * dx + (py - sy) * dy) / (dx * dx + dy * dy) - t = max(0.0, min(1.0, t)) - cx = sx + t * dx - cy = sy + t * dy - return math.hypot(px - cx, py - cy) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/connectors/opensky/README.md b/connectors/opensky/README.md index d1a95d0..7899688 100644 --- a/connectors/opensky/README.md +++ b/connectors/opensky/README.md @@ -21,10 +21,6 @@ Alternate built-in presets: - `airport_fences.py`: creates airport and apron-sector zones/fences from presets - `opensky_support.py`: OpenSky polling and preset helpers - `hub_client.py`: hub REST and WebSocket helpers -- `scripts/log_locations.py`: subscribes to `location_updates` and writes NDJSON -- `scripts/log_fence_events.py`: subscribes to `fence_events` and writes NDJSON -- `scripts/log_collision_events.py`: subscribes to `collision_events` and writes NDJSON -- `scripts/check_fence_alignment.py`: compares logged aircraft locations against current hub fences - `.env.example`: environment template - `pyproject.toml`: `uv`-managed Python project metadata - `uv.lock`: locked Python dependency set @@ -92,18 +88,18 @@ uv run --project connectors/opensky python connectors/opensky/airport_fences.py uv run --project connectors/opensky python connectors/opensky/connector.py --env-file connectors/opensky/.env.local ``` -7. Optional: log live WebSocket topics: +7. Optional: log live WebSocket topics with the shared root scripts: ```bash -uv run --project connectors/opensky python connectors/opensky/scripts/log_locations.py --env-file connectors/opensky/.env.local -uv run --project connectors/opensky python connectors/opensky/scripts/log_fence_events.py --env-file connectors/opensky/.env.local -uv run --project connectors/opensky python connectors/opensky/scripts/log_collision_events.py --env-file connectors/opensky/.env.local +uv run --project scripts python scripts/log_locations.py --env-file connectors/opensky/.env.local --output connectors/opensky/logs/location_updates.ndjson +uv run --project scripts python scripts/log_fence_events.py --env-file connectors/opensky/.env.local --output connectors/opensky/logs/fence_events.ndjson +uv run --project scripts python scripts/log_collision_events.py --env-file connectors/opensky/.env.local --output connectors/opensky/logs/collision_events.ndjson ``` 8. Check how close captured aircraft positions came to the airport fences: ```bash -uv run --project connectors/opensky python connectors/opensky/scripts/check_fence_alignment.py --env-file connectors/opensky/.env.local +uv run --project scripts python scripts/check_fence_alignment.py --env-file connectors/opensky/.env.local --locations-log connectors/opensky/logs/location_updates.ndjson ``` ## Hub Mapping diff --git a/connectors/opensky/scripts/log_collision_events.py b/connectors/opensky/scripts/log_collision_events.py deleted file mode 100644 index 7abda90..0000000 --- a/connectors/opensky/scripts/log_collision_events.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Log collision_events messages to an NDJSON file.""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", default="connectors/opensky/logs/collision_events.ndjson") - parser.add_argument("--env-file", default="connectors/opensky/.env.local") - args = parser.parse_args() - logger = Path(__file__).with_name("ws_ndjson_logger.py") - command = [sys.executable, str(logger), "--topic", "collision_events", "--output", args.output, "--env-file", args.env_file] - try: - return subprocess.call(command) - except KeyboardInterrupt: - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/connectors/opensky/scripts/log_fence_events.py b/connectors/opensky/scripts/log_fence_events.py deleted file mode 100644 index da54acf..0000000 --- a/connectors/opensky/scripts/log_fence_events.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Log fence_events messages to an NDJSON file.""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", default="connectors/opensky/logs/fence_events.ndjson") - parser.add_argument("--env-file", default="connectors/opensky/.env.local") - args = parser.parse_args() - logger = Path(__file__).with_name("ws_ndjson_logger.py") - command = [sys.executable, str(logger), "--topic", "fence_events", "--output", args.output, "--env-file", args.env_file] - try: - return subprocess.call(command) - except KeyboardInterrupt: - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/connectors/opensky/scripts/log_locations.py b/connectors/opensky/scripts/log_locations.py deleted file mode 100644 index 753be1b..0000000 --- a/connectors/opensky/scripts/log_locations.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Log location_updates messages to an NDJSON file.""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", default="connectors/opensky/logs/location_updates.ndjson") - parser.add_argument("--env-file", default="connectors/opensky/.env.local") - args = parser.parse_args() - logger = Path(__file__).with_name("ws_ndjson_logger.py") - command = [sys.executable, str(logger), "--topic", "location_updates", "--output", args.output, "--env-file", args.env_file] - try: - return subprocess.call(command) - except KeyboardInterrupt: - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/connectors/opensky/scripts/ws_ndjson_logger.py b/connectors/opensky/scripts/ws_ndjson_logger.py deleted file mode 100644 index be27b15..0000000 --- a/connectors/opensky/scripts/ws_ndjson_logger.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -"""Subscribe to one hub WebSocket topic and append wrapper events as NDJSON.""" - -from __future__ import annotations - -import argparse -import json -import os -import signal -import sys -import time -from datetime import datetime, timezone -from pathlib import Path - -import websocket - -SCRIPT_DIR = Path(__file__).resolve().parent -ROOT_DIR = SCRIPT_DIR.parent -sys.path.insert(0, str(ROOT_DIR)) - -from opensky_support import load_env_file # noqa: E402 - - -RUNNING = True - - -def build_argument_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--topic", required=True) - parser.add_argument("--output", required=True) - parser.add_argument("--env-file", default=os.getenv("OPENSKY_ENV_FILE")) - parser.add_argument("--ws-url", default=os.getenv("HUB_WS_URL")) - parser.add_argument("--token", default=os.getenv("HUB_TOKEN")) - return parser - - -def main() -> int: - global RUNNING - args = build_argument_parser().parse_args() - load_env_file(args.env_file) - ws_url = args.ws_url or os.getenv("HUB_WS_URL") - if not ws_url: - raise SystemExit("HUB_WS_URL or --ws-url is required") - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - signal.signal(signal.SIGINT, handle_stop) - signal.signal(signal.SIGTERM, handle_stop) - - token = args.token or os.getenv("HUB_TOKEN") - with output_path.open("a", encoding="utf-8") as handle: - while RUNNING: - connection: websocket.WebSocket | None = None - try: - connection = connect_and_subscribe(ws_url, args.topic, token) - while RUNNING: - try: - raw = connection.recv() - except TimeoutError: - connection.ping("keepalive") - continue - except websocket.WebSocketTimeoutException: - connection.ping("keepalive") - continue - payload = { - "received_at": datetime.now(timezone.utc).isoformat(), - "topic": args.topic, - "message": json.loads(raw), - } - handle.write(json.dumps(payload, separators=(",", ":")) + "\n") - handle.flush() - time.sleep(0.01) - except KeyboardInterrupt: - break - except Exception: - time.sleep(2.0) - finally: - if connection is not None: - connection.close() - return 0 - - -def handle_stop(_signum: int, _frame: object) -> None: - global RUNNING - RUNNING = False - - -def connect_and_subscribe(ws_url: str, topic: str, token: str | None) -> websocket.WebSocket: - connection = websocket.create_connection(ws_url, timeout=30) - connection.settimeout(15) - subscribe = {"event": "subscribe", "topic": topic} - if token: - subscribe["params"] = {"token": token} - connection.send(json.dumps(subscribe)) - return connection - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/connectors/replay/.env.example b/connectors/replay/.env.example new file mode 100644 index 0000000..3204874 --- /dev/null +++ b/connectors/replay/.env.example @@ -0,0 +1,9 @@ +HUB_HTTP_URL=http://localhost:8080 +HUB_WS_URL=ws://localhost:8080/v2/ws/socket +HUB_TOKEN= + +REPLAY_INPUT=connectors/gtfs/logs/location_updates.ndjson +REPLAY_ACCELERATION_FACTOR=1.0 +REPLAY_INTERPOLATION_RATE_HZ=0.0 + +LOG_LEVEL=INFO diff --git a/connectors/replay/README.md b/connectors/replay/README.md new file mode 100644 index 0000000..9454a5e --- /dev/null +++ b/connectors/replay/README.md @@ -0,0 +1,173 @@ +# Replay Connector + +This connector is a diagnostic tool for replaying logged hub +`location_updates` NDJSON back into an Open RTLS Hub. It is intended for trace +files produced by the shared root logging scripts such as +[`scripts/log_locations.py`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_locations.py). + +Replay behavior: + +- preserves the relative timing between captured location updates +- shifts the full stream so the first emitted timestamp starts at "now" +- supports real-time replay or faster playback via `--acceleration-factor` +- can emit synthetic straight-line interpolation points per object via + `--interpolation-rate-hz` +- best-effort bootstraps referenced providers and trackables when `HUB_HTTP_URL` + is configured + +## Files + +- `connector.py`: loads NDJSON, corrects timestamps to the current replay time, + and publishes the stream over `location_updates` +- `replay_support.py`: NDJSON parsing, replay scheduling, and interpolation helpers +- `hub_client.py`: hub REST and WebSocket helpers +- `.env.example`: environment template +- `pyproject.toml`: `uv`-managed Python project metadata + +## Shared Local Hub + +Start the reusable local hub runtime first: + +```bash +connectors/local-hub/start_demo.sh +``` + +Fetch an admin token when auth is enabled: + +```bash +connectors/local-hub/fetch_demo_token.sh +``` + +## Required Inputs + +- `HUB_WS_URL`: WebSocket endpoint such as `ws://localhost:8080/v2/ws/socket` +- `REPLAY_INPUT`: path to a logged `location_updates` NDJSON file + +Optional but recommended: + +- `HUB_HTTP_URL`: enables best-effort provider and trackable bootstrap before replay +- `HUB_TOKEN`: bearer token when hub auth is enabled +- `REPLAY_ACCELERATION_FACTOR`: playback speed multiplier, where `1.0` is real time +- `REPLAY_INTERPOLATION_RATE_HZ`: per-object interpolation cadence in Hertz, where + `1.0` means once per second and `0.1` means once every 10 seconds + +## Setup + +1. Start the shared local hub: + +```bash +connectors/local-hub/start_demo.sh +``` + +2. Copy `.env.example` to `.env.local`: + +```bash +cp connectors/replay/.env.example connectors/replay/.env.local +``` + +3. Set `REPLAY_INPUT` to a previously captured NDJSON trace. For example, first + capture a hub location feed: + +```bash +uv run --project scripts python scripts/log_locations.py --env-file connectors/gtfs/.env.local --output connectors/gtfs/logs/location_updates.ndjson +``` + +4. Sync the Python runtime: + +```bash +uv sync --project connectors/replay +``` + +5. Start the replay connector: + +```bash +uv run --project connectors/replay python connectors/replay/connector.py --env-file connectors/replay/.env.local +``` + +## Playback Examples + +Replay in real time: + +```bash +uv run --project connectors/replay python connectors/replay/connector.py \ + --env-file connectors/replay/.env.local \ + --input connectors/gtfs/logs/location_updates.ndjson +``` + +Replay four times faster: + +```bash +uv run --project connectors/replay python connectors/replay/connector.py \ + --env-file connectors/replay/.env.local \ + --input connectors/gtfs/logs/location_updates.ndjson \ + --acceleration-factor 4.0 +``` + +Replay with interpolated points every second: + +```bash +uv run --project connectors/replay python connectors/replay/connector.py \ + --env-file connectors/replay/.env.local \ + --input connectors/opensky/logs/location_updates.ndjson \ + --interpolation-rate-hz 1.0 +``` + +Replay with interpolated points every 10 seconds: + +```bash +uv run --project connectors/replay python connectors/replay/connector.py \ + --env-file connectors/replay/.env.local \ + --input connectors/opensky/logs/location_updates.ndjson \ + --interpolation-rate-hz 0.1 +``` + +## Input Format + +The connector expects the NDJSON shape produced by the shared logging scripts: + +```json +{ + "received_at": "2026-04-02T10:15:00+00:00", + "topic": "location_updates", + "message": { + "event": "message", + "topic": "location_updates", + "payload": [ + { + "position": { "type": "Point", "coordinates": [8.56, 50.03] }, + "provider_id": "opensky-demo", + "provider_type": "adsb", + "source": "opensky:3c6621", + "timestamp_generated": "2026-04-02T10:14:58+00:00" + } + ] + } +} +``` + +Replay uses `payload[*].timestamp_generated` when present and falls back to +`received_at` otherwise. + +## Interpolation Behavior + +Interpolation is keyed per object. The connector uses the first trackable ID +when present and otherwise falls back to the location `source`. For each object, +synthetic points are inserted on a straight line between two consecutive logged +positions when the elapsed time between them is greater than the requested +interval. + +Synthetic locations: + +- inherit the later sample as their metadata template +- receive an adjusted `timestamp_generated` aligned to the replay clock +- include `properties.replay_synthetic_interpolation=true` +- preserve the original source timestamp in + `properties.replay_original_timestamp_generated` + +## Limitations + +- interpolation is linear in WGS84 longitude and latitude; it is intended for + diagnostic playback, not precise route reconstruction +- only GeoJSON `Point` locations are supported +- replay can recreate provider and trackable IDs, but it cannot restore + metadata that was never present in the logged location payload diff --git a/connectors/replay/connector.py b/connectors/replay/connector.py new file mode 100644 index 0000000..f20829f --- /dev/null +++ b/connectors/replay/connector.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Replay logged location NDJSON files into a local Open RTLS Hub.""" + +from __future__ import annotations + +import argparse +import logging +import os +import time +from datetime import UTC, datetime + +from hub_client import HubConfig, HubRESTClient, HubWebSocketPublisher +from replay_support import build_replay_schedule, load_env_file, load_logged_locations + + +LOGGER = logging.getLogger("replay.connector") + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", default=os.getenv("REPLAY_INPUT"), help="Path to a location_updates NDJSON file") + parser.add_argument("--env-file", default=os.getenv("REPLAY_ENV_FILE")) + parser.add_argument( + "--acceleration-factor", + type=float, + default=float(os.getenv("REPLAY_ACCELERATION_FACTOR", "1.0")), + help="Replay speed multiplier. 1.0 is real time, 2.0 is twice as fast.", + ) + parser.add_argument( + "--interpolation-rate-hz", + type=float, + default=float(os.getenv("REPLAY_INTERPOLATION_RATE_HZ", "0.0")), + help="Synthetic interpolation cadence per object in Hertz. 1.0 emits once per second.", + ) + return parser + + +def main() -> int: + args = build_argument_parser().parse_args() + load_env_file(args.env_file) + + if not args.input: + raise SystemExit("--input or REPLAY_INPUT is required") + if args.acceleration_factor <= 0: + raise SystemExit("--acceleration-factor must be greater than 0") + if args.interpolation_rate_hz < 0: + raise SystemExit("--interpolation-rate-hz must be greater than or equal to 0") + + logging.basicConfig( + level=os.getenv("LOG_LEVEL", "INFO").upper(), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + hub_config = HubConfig( + http_url=(os.getenv("HUB_HTTP_URL") or "").strip() or None, + ws_url=require_env("HUB_WS_URL"), + token=os.getenv("HUB_TOKEN") or None, + ) + hub_rest = HubRESTClient(hub_config) + hub_ws = HubWebSocketPublisher(hub_config) + + logged_locations = load_logged_locations(args.input) + replay_start = datetime.now(UTC) + replay_schedule = build_replay_schedule( + logged_locations=logged_locations, + replay_start=replay_start, + acceleration_factor=args.acceleration_factor, + interpolation_rate_hz=args.interpolation_rate_hz, + ) + + LOGGER.info( + "loaded %d logged locations and scheduled %d replay emissions starting at %s", + len(logged_locations), + len(replay_schedule), + replay_start.isoformat(), + ) + + ensure_hub_resources(hub_rest, replay_schedule) + + start_monotonic = time.monotonic() + try: + for event in replay_schedule: + wait_until_scheduled(start_monotonic, replay_start, event.replay_timestamp) + hub_ws.publish_locations([event.location]) + LOGGER.debug( + "published replay event synthetic=%s source=%s timestamp=%s", + event.synthetic, + event.location.get("source"), + event.location.get("timestamp_generated"), + ) + except KeyboardInterrupt: + LOGGER.info("stopping replay connector") + return 0 + finally: + hub_ws.close() + + LOGGER.info("replayed %d location updates from %s", len(replay_schedule), args.input) + return 0 + + +def ensure_hub_resources(hub_rest: HubRESTClient, replay_schedule: list) -> None: + if not hub_rest.config.http_url: + LOGGER.info("HUB_HTTP_URL not set; skipping provider and trackable bootstrap") + return + + known_providers: set[str] = set() + known_trackables: set[str] = set() + for event in replay_schedule: + location = event.location + provider_id = location.get("provider_id") + provider_type = location.get("provider_type") or "replay" + if isinstance(provider_id, str) and provider_id and provider_id not in known_providers: + hub_rest.ensure_provider( + provider_id=provider_id, + provider_type=str(provider_type), + name=provider_id, + properties={"connector": "replay"}, + ) + known_providers.add(provider_id) + + trackables = location.get("trackables") + if not isinstance(trackables, list) or not isinstance(provider_id, str) or not provider_id: + continue + for trackable_id in trackables: + if not isinstance(trackable_id, str) or not trackable_id or trackable_id in known_trackables: + continue + hub_rest.ensure_trackable( + trackable_id=trackable_id, + name=trackable_name(location, trackable_id), + provider_id=provider_id, + properties=trackable_properties(location), + ) + known_trackables.add(trackable_id) + + +def trackable_name(location: dict[str, object], trackable_id: str) -> str: + properties = location.get("properties") + if isinstance(properties, dict): + for key in ("vehicle_label", "vehicle_id", "callsign", "icao24", "external_vehicle_id"): + value = properties.get(key) + if isinstance(value, str) and value: + return value + source = location.get("source") + if isinstance(source, str) and source: + return source + return trackable_id + + +def trackable_properties(location: dict[str, object]) -> dict[str, object]: + properties = location.get("properties") + if isinstance(properties, dict): + merged = dict(properties) + else: + merged = {} + merged["connector"] = "replay" + return merged + + +def wait_until_scheduled(start_monotonic: float, replay_start: datetime, replay_timestamp: datetime) -> None: + delay_seconds = (replay_timestamp - replay_start).total_seconds() + target_monotonic = start_monotonic + max(delay_seconds, 0.0) + while True: + remaining = target_monotonic - time.monotonic() + if remaining <= 0: + return + time.sleep(min(remaining, 0.25)) + + +def require_env(name: str) -> str: + value = os.getenv(name) + if not value: + raise SystemExit(f"{name} is required") + return value + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/connectors/replay/hub_client.py b/connectors/replay/hub_client.py new file mode 100644 index 0000000..833118f --- /dev/null +++ b/connectors/replay/hub_client.py @@ -0,0 +1,166 @@ +"""Helpers for talking to a local Open RTLS Hub from connector scripts.""" + +from __future__ import annotations + +import json +import logging +import threading +from dataclasses import dataclass +from typing import Any +from urllib.parse import urljoin + +import requests +import websocket + + +LOGGER = logging.getLogger(__name__) + + +@dataclass +class HubConfig: + ws_url: str + http_url: str | None = None + token: str | None = None + timeout_seconds: float = 30.0 + + +class HubRESTClient: + """Best-effort idempotent CRUD helpers for connector-managed resources.""" + + def __init__(self, config: HubConfig): + self.config = config + self.session = requests.Session() + self.session.headers.update({"Accept": "application/json"}) + if config.token: + self.session.headers.update({"Authorization": f"Bearer {config.token}"}) + + def ensure_provider( + self, + provider_id: str, + provider_type: str, + name: str, + properties: dict[str, Any] | None = None, + ) -> dict[str, Any]: + if not self.config.http_url: + return {} + payload = { + "id": provider_id, + "type": provider_type, + "name": name, + "properties": properties or {}, + } + return self._ensure_resource("/v2/providers", f"/v2/providers/{provider_id}", payload) + + def ensure_trackable( + self, + trackable_id: str, + name: str, + provider_id: str, + properties: dict[str, Any] | None = None, + ) -> dict[str, Any]: + if not self.config.http_url: + return {} + payload = { + "id": trackable_id, + "type": "virtual", + "name": name, + "location_providers": [provider_id], + "properties": properties or {}, + } + return self._ensure_resource("/v2/trackables", f"/v2/trackables/{trackable_id}", payload) + + def _ensure_resource(self, collection_path: str, item_path: str, payload: dict[str, Any]) -> dict[str, Any]: + existing = self._request("GET", item_path, expected={200, 404}) + if existing.status_code == 404: + return self._request("POST", collection_path, json_body=payload, expected={201}).json() + return self._request("PUT", item_path, json_body=payload, expected={200}).json() + + def _request( + self, + method: str, + path: str, + json_body: dict[str, Any] | None = None, + expected: set[int] | None = None, + ) -> requests.Response: + if not self.config.http_url: + raise RuntimeError("Hub REST operations require HUB_HTTP_URL") + response = self.session.request( + method=method, + url=urljoin(self.config.http_url.rstrip("/") + "/", path.lstrip("/")), + json=json_body, + timeout=self.config.timeout_seconds, + ) + if expected and response.status_code not in expected: + try: + details = response.json() + except ValueError: + details = response.text + raise RuntimeError(f"{method} {path} returned {response.status_code}: {details}") + return response + + +class HubWebSocketPublisher: + """Send OMLOX wrapper messages to the hub over WebSocket.""" + + def __init__(self, config: HubConfig): + self.config = config + self._connection: websocket.WebSocket | None = None + self._lock = threading.Lock() + self._keepalive_stop = threading.Event() + self._keepalive_thread = threading.Thread(target=self._keepalive_loop, name="hub-ws-keepalive", daemon=True) + self._keepalive_thread.start() + + def close(self) -> None: + self._keepalive_stop.set() + with self._lock: + self._close_locked() + self._keepalive_thread.join(timeout=1.0) + + def publish_locations(self, locations: list[dict[str, Any]]) -> None: + if not locations: + return + message: dict[str, Any] = { + "event": "message", + "topic": "location_updates", + "payload": locations, + } + if self.config.token: + message["params"] = {"token": self.config.token} + self._send(json.dumps(message)) + + def _connect(self) -> websocket.WebSocket: + LOGGER.info("connecting websocket publisher to %s", self.config.ws_url) + connection = websocket.create_connection(self.config.ws_url, timeout=self.config.timeout_seconds) + connection.settimeout(self.config.timeout_seconds) + return connection + + def _send(self, raw: str) -> None: + with self._lock: + if self._connection is None: + self._connection = self._connect() + try: + self._connection.send(raw) + except Exception: + LOGGER.info("websocket send failed; reconnecting") + self._close_locked() + self._connection = self._connect() + self._connection.send(raw) + + def _keepalive_loop(self) -> None: + while not self._keepalive_stop.wait(15.0): + with self._lock: + if self._connection is None: + continue + try: + self._connection.ping("keepalive") + except Exception: + LOGGER.info("websocket ping failed; reconnecting on next send") + self._close_locked() + + def _close_locked(self) -> None: + if self._connection is not None: + try: + self._connection.close() + except Exception: + LOGGER.debug("websocket close failed", exc_info=True) + self._connection = None diff --git a/connectors/replay/pyproject.toml b/connectors/replay/pyproject.toml new file mode 100644 index 0000000..fb3b744 --- /dev/null +++ b/connectors/replay/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "open-rtls-replay-demo" +version = "0.1.0" +description = "NDJSON replay connector for Open RTLS Hub" +requires-python = ">=3.12" +dependencies = [ + "requests>=2.33.0", + "websocket-client>=1.9.0", +] + +[tool.uv] +package = false diff --git a/connectors/replay/replay_support.py b/connectors/replay/replay_support.py new file mode 100644 index 0000000..1dbd440 --- /dev/null +++ b/connectors/replay/replay_support.py @@ -0,0 +1,242 @@ +"""Helpers for replaying logged hub location NDJSON streams.""" + +from __future__ import annotations + +import copy +import json +import os +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + + +def load_env_file(path: str | None) -> None: + if not path or not os.path.exists(path): + return + with open(path, "r", encoding="utf-8") as handle: + for line in handle: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + os.environ.setdefault(key.strip(), value.strip()) + + +@dataclass(frozen=True) +class LoggedLocation: + order: int + timestamp: datetime + location: dict[str, Any] + + +@dataclass(frozen=True) +class ReplayLocation: + order: float + original_timestamp: datetime + replay_timestamp: datetime + location: dict[str, Any] + synthetic: bool + + +def load_logged_locations(path: str) -> list[LoggedLocation]: + logged_locations: list[LoggedLocation] = [] + input_path = Path(path) + with input_path.open("r", encoding="utf-8") as handle: + for line_number, raw_line in enumerate(handle, start=1): + stripped = raw_line.strip() + if not stripped: + continue + wrapper = json.loads(stripped) + message = wrapper.get("message") + if not isinstance(message, dict): + continue + payload = message.get("payload") + if not isinstance(payload, list): + continue + fallback_timestamp = parse_timestamp(wrapper.get("received_at")) + for payload_index, raw_location in enumerate(payload): + if not isinstance(raw_location, dict): + continue + location = copy.deepcopy(raw_location) + timestamp = parse_timestamp(location.get("timestamp_generated")) or fallback_timestamp + if timestamp is None: + raise ValueError(f"{path}:{line_number} is missing both timestamp_generated and received_at") + logged_locations.append( + LoggedLocation( + order=(line_number * 1000) + payload_index, + timestamp=timestamp, + location=location, + ) + ) + if not logged_locations: + raise ValueError(f"{path} did not contain any replayable location payloads") + return sorted(logged_locations, key=lambda item: (item.timestamp, item.order)) + + +def build_replay_schedule( + logged_locations: list[LoggedLocation], + replay_start: datetime, + acceleration_factor: float, + interpolation_rate_hz: float, +) -> list[ReplayLocation]: + if acceleration_factor <= 0: + raise ValueError("acceleration_factor must be greater than 0") + if interpolation_rate_hz < 0: + raise ValueError("interpolation_rate_hz must be greater than or equal to 0") + + expanded = interpolate_logged_locations(logged_locations, interpolation_rate_hz) + baseline = expanded[0].timestamp + + replay_schedule: list[ReplayLocation] = [] + for item in expanded: + replay_offset = (item.timestamp - baseline).total_seconds() / acceleration_factor + replay_timestamp = replay_start + timedelta(seconds=replay_offset) + replay_schedule.append( + ReplayLocation( + order=float(item.order), + original_timestamp=item.timestamp, + replay_timestamp=replay_timestamp, + location=prepare_location_for_replay(item.location, item.timestamp, replay_timestamp, item.synthetic), + synthetic=item.synthetic, + ) + ) + return replay_schedule + + +@dataclass(frozen=True) +class ExpandedLocation: + order: float + timestamp: datetime + location: dict[str, Any] + synthetic: bool + + +def interpolate_logged_locations( + logged_locations: list[LoggedLocation], + interpolation_rate_hz: float, +) -> list[ExpandedLocation]: + if interpolation_rate_hz <= 0: + return [ + ExpandedLocation( + order=float(item.order), + timestamp=item.timestamp, + location=item.location, + synthetic=False, + ) + for item in logged_locations + ] + + interval_seconds = 1.0 / interpolation_rate_hz + expanded: list[ExpandedLocation] = [] + previous_by_key: dict[str, LoggedLocation] = {} + + for item in logged_locations: + key = replay_object_key(item.location) + previous = previous_by_key.get(key) + if previous is not None: + elapsed = (item.timestamp - previous.timestamp).total_seconds() + if elapsed > interval_seconds: + step_count = int(elapsed / interval_seconds) + for step in range(1, step_count + 1): + interpolated_offset = step * interval_seconds + if interpolated_offset >= elapsed: + break + fraction = interpolated_offset / elapsed + interpolated_timestamp = previous.timestamp + timedelta(seconds=interpolated_offset) + expanded.append( + ExpandedLocation( + order=previous.order + fraction, + timestamp=interpolated_timestamp, + location=interpolate_location(previous.location, item.location, fraction), + synthetic=True, + ) + ) + expanded.append( + ExpandedLocation( + order=float(item.order), + timestamp=item.timestamp, + location=item.location, + synthetic=False, + ) + ) + previous_by_key[key] = item + + return sorted(expanded, key=lambda item: (item.timestamp, item.order)) + + +def interpolate_location(previous: dict[str, Any], current: dict[str, Any], fraction: float) -> dict[str, Any]: + previous_coordinates = coordinates(previous) + current_coordinates = coordinates(current) + + synthetic = copy.deepcopy(current) + synthetic["position"] = { + "type": "Point", + "coordinates": [ + interpolate_float(previous_coordinates[0], current_coordinates[0], fraction), + interpolate_float(previous_coordinates[1], current_coordinates[1], fraction), + ], + } + if previous.get("speed") is not None and current.get("speed") is not None: + synthetic["speed"] = interpolate_float(float(previous["speed"]), float(current["speed"]), fraction) + if previous.get("course") is not None and current.get("course") is not None: + synthetic["course"] = interpolate_float(float(previous["course"]), float(current["course"]), fraction) + return synthetic + + +def prepare_location_for_replay( + location: dict[str, Any], + original_timestamp: datetime, + replay_timestamp: datetime, + synthetic: bool, +) -> dict[str, Any]: + replay_location = copy.deepcopy(location) + replay_location["timestamp_generated"] = replay_timestamp.astimezone(UTC).isoformat() + properties = replay_location.setdefault("properties", {}) + if isinstance(properties, dict): + properties["replay_original_timestamp_generated"] = original_timestamp.astimezone(UTC).isoformat() + properties["replay_synthetic_interpolation"] = synthetic + return replay_location + + +def replay_object_key(location: dict[str, Any]) -> str: + trackables = location.get("trackables") + if isinstance(trackables, list) and trackables: + return f"trackable:{trackables[0]}" + source = location.get("source") + if isinstance(source, str) and source: + return f"source:{source}" + provider_id = location.get("provider_id") or "provider" + coordinates_value = coordinates(location) + return f"fallback:{provider_id}:{coordinates_value[0]}:{coordinates_value[1]}" + + +def coordinates(location: dict[str, Any]) -> tuple[float, float]: + position = location.get("position") + if not isinstance(position, dict) or position.get("type") != "Point": + raise ValueError("replay connector only supports Point locations") + coordinates_value = position.get("coordinates") + if ( + not isinstance(coordinates_value, list) + or len(coordinates_value) < 2 + or coordinates_value[0] is None + or coordinates_value[1] is None + ): + raise ValueError("location is missing valid Point coordinates") + return float(coordinates_value[0]), float(coordinates_value[1]) + + +def parse_timestamp(raw_value: Any) -> datetime | None: + if not isinstance(raw_value, str) or not raw_value: + return None + normalized = raw_value.strip() + if normalized.endswith("Z"): + normalized = normalized[:-1] + "+00:00" + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None: + return parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def interpolate_float(previous: float, current: float, fraction: float) -> float: + return previous + ((current - previous) * fraction) diff --git a/connectors/replay/uv.lock b/connectors/replay/uv.lock new file mode 100644 index 0000000..f8850d1 --- /dev/null +++ b/connectors/replay/uv.lock @@ -0,0 +1,142 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "certifi" +version = "2026.2.25" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/60/e3bec1881450851b087e301bedc3daa9377a4d45f1c26aa90b0b235e38aa/charset_normalizer-3.4.6.tar.gz", hash = "sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6", size = 143363, upload-time = "2026-03-15T18:53:25.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/62/c0815c992c9545347aeea7859b50dc9044d147e2e7278329c6e02ac9a616/charset_normalizer-3.4.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab", size = 295154, upload-time = "2026-03-15T18:50:50.88Z" }, + { url = "https://files.pythonhosted.org/packages/a8/37/bdca6613c2e3c58c7421891d80cc3efa1d32e882f7c4a7ee6039c3fc951a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21", size = 199191, upload-time = "2026-03-15T18:50:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/6c/92/9934d1bbd69f7f398b38c5dae1cbf9cc672e7c34a4adf7b17c0a9c17d15d/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2", size = 218674, upload-time = "2026-03-15T18:50:54.102Z" }, + { url = "https://files.pythonhosted.org/packages/af/90/25f6ab406659286be929fd89ab0e78e38aa183fc374e03aa3c12d730af8a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff", size = 215259, upload-time = "2026-03-15T18:50:55.616Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ef/79a463eb0fff7f96afa04c1d4c51f8fc85426f918db467854bfb6a569ce3/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5", size = 207276, upload-time = "2026-03-15T18:50:57.054Z" }, + { url = "https://files.pythonhosted.org/packages/f7/72/d0426afec4b71dc159fa6b4e68f868cd5a3ecd918fec5813a15d292a7d10/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0", size = 195161, upload-time = "2026-03-15T18:50:58.686Z" }, + { url = "https://files.pythonhosted.org/packages/bf/18/c82b06a68bfcb6ce55e508225d210c7e6a4ea122bfc0748892f3dc4e8e11/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a", size = 203452, upload-time = "2026-03-15T18:51:00.196Z" }, + { url = "https://files.pythonhosted.org/packages/44/d6/0c25979b92f8adafdbb946160348d8d44aa60ce99afdc27df524379875cb/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2", size = 202272, upload-time = "2026-03-15T18:51:01.703Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/7fea3e8fe84136bebbac715dd1221cc25c173c57a699c030ab9b8900cbb7/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5", size = 195622, upload-time = "2026-03-15T18:51:03.526Z" }, + { url = "https://files.pythonhosted.org/packages/57/8a/d6f7fd5cb96c58ef2f681424fbca01264461336d2a7fc875e4446b1f1346/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6", size = 220056, upload-time = "2026-03-15T18:51:05.269Z" }, + { url = "https://files.pythonhosted.org/packages/16/50/478cdda782c8c9c3fb5da3cc72dd7f331f031e7f1363a893cdd6ca0f8de0/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d", size = 203751, upload-time = "2026-03-15T18:51:06.858Z" }, + { url = "https://files.pythonhosted.org/packages/75/fc/cc2fcac943939c8e4d8791abfa139f685e5150cae9f94b60f12520feaa9b/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2", size = 216563, upload-time = "2026-03-15T18:51:08.564Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b7/a4add1d9a5f68f3d037261aecca83abdb0ab15960a3591d340e829b37298/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923", size = 209265, upload-time = "2026-03-15T18:51:10.312Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/c094561b5d64a24277707698e54b7f67bd17a4f857bbfbb1072bba07c8bf/charset_normalizer-3.4.6-cp312-cp312-win32.whl", hash = "sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4", size = 144229, upload-time = "2026-03-15T18:51:11.694Z" }, + { url = "https://files.pythonhosted.org/packages/ab/20/0567efb3a8fd481b8f34f739ebddc098ed062a59fed41a8d193a61939e8f/charset_normalizer-3.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb", size = 154277, upload-time = "2026-03-15T18:51:13.004Z" }, + { url = "https://files.pythonhosted.org/packages/15/57/28d79b44b51933119e21f65479d0864a8d5893e494cf5daab15df0247c17/charset_normalizer-3.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4", size = 142817, upload-time = "2026-03-15T18:51:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1d/4fdabeef4e231153b6ed7567602f3b68265ec4e5b76d6024cf647d43d981/charset_normalizer-3.4.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f", size = 294823, upload-time = "2026-03-15T18:51:15.755Z" }, + { url = "https://files.pythonhosted.org/packages/47/7b/20e809b89c69d37be748d98e84dce6820bf663cf19cf6b942c951a3e8f41/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843", size = 198527, upload-time = "2026-03-15T18:51:17.177Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/4f8d27527d59c039dce6f7622593cdcd3d70a8504d87d09eb11e9fdc6062/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf", size = 218388, upload-time = "2026-03-15T18:51:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/f6/9b/4770ccb3e491a9bacf1c46cc8b812214fe367c86a96353ccc6daf87b01ec/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8", size = 214563, upload-time = "2026-03-15T18:51:20.374Z" }, + { url = "https://files.pythonhosted.org/packages/2b/58/a199d245894b12db0b957d627516c78e055adc3a0d978bc7f65ddaf7c399/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9", size = 206587, upload-time = "2026-03-15T18:51:21.807Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/3def227f1ec56f5c69dfc8392b8bd63b11a18ca8178d9211d7cc5e5e4f27/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88", size = 194724, upload-time = "2026-03-15T18:51:23.508Z" }, + { url = "https://files.pythonhosted.org/packages/58/ab/9318352e220c05efd31c2779a23b50969dc94b985a2efa643ed9077bfca5/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84", size = 202956, upload-time = "2026-03-15T18:51:25.239Z" }, + { url = "https://files.pythonhosted.org/packages/75/13/f3550a3ac25b70f87ac98c40d3199a8503676c2f1620efbf8d42095cfc40/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd", size = 201923, upload-time = "2026-03-15T18:51:26.682Z" }, + { url = "https://files.pythonhosted.org/packages/1b/db/c5c643b912740b45e8eec21de1bbab8e7fc085944d37e1e709d3dcd9d72f/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c", size = 195366, upload-time = "2026-03-15T18:51:28.129Z" }, + { url = "https://files.pythonhosted.org/packages/5a/67/3b1c62744f9b2448443e0eb160d8b001c849ec3fef591e012eda6484787c/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194", size = 219752, upload-time = "2026-03-15T18:51:29.556Z" }, + { url = "https://files.pythonhosted.org/packages/f6/98/32ffbaf7f0366ffb0445930b87d103f6b406bc2c271563644bde8a2b1093/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc", size = 203296, upload-time = "2026-03-15T18:51:30.921Z" }, + { url = "https://files.pythonhosted.org/packages/41/12/5d308c1bbe60cabb0c5ef511574a647067e2a1f631bc8634fcafaccd8293/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f", size = 215956, upload-time = "2026-03-15T18:51:32.399Z" }, + { url = "https://files.pythonhosted.org/packages/53/e9/5f85f6c5e20669dbe56b165c67b0260547dea97dba7e187938833d791687/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2", size = 208652, upload-time = "2026-03-15T18:51:34.214Z" }, + { url = "https://files.pythonhosted.org/packages/f1/11/897052ea6af56df3eef3ca94edafee410ca699ca0c7b87960ad19932c55e/charset_normalizer-3.4.6-cp313-cp313-win32.whl", hash = "sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d", size = 143940, upload-time = "2026-03-15T18:51:36.15Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5c/724b6b363603e419829f561c854b87ed7c7e31231a7908708ac086cdf3e2/charset_normalizer-3.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389", size = 154101, upload-time = "2026-03-15T18:51:37.876Z" }, + { url = "https://files.pythonhosted.org/packages/01/a5/7abf15b4c0968e47020f9ca0935fb3274deb87cb288cd187cad92e8cdffd/charset_normalizer-3.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f", size = 143109, upload-time = "2026-03-15T18:51:39.565Z" }, + { url = "https://files.pythonhosted.org/packages/25/6f/ffe1e1259f384594063ea1869bfb6be5cdb8bc81020fc36c3636bc8302a1/charset_normalizer-3.4.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8", size = 294458, upload-time = "2026-03-15T18:51:41.134Z" }, + { url = "https://files.pythonhosted.org/packages/56/60/09bb6c13a8c1016c2ed5c6a6488e4ffef506461aa5161662bd7636936fb1/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421", size = 199277, upload-time = "2026-03-15T18:51:42.953Z" }, + { url = "https://files.pythonhosted.org/packages/00/50/dcfbb72a5138bbefdc3332e8d81a23494bf67998b4b100703fd15fa52d81/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2", size = 218758, upload-time = "2026-03-15T18:51:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/03/b3/d79a9a191bb75f5aa81f3aaaa387ef29ce7cb7a9e5074ba8ea095cc073c2/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30", size = 215299, upload-time = "2026-03-15T18:51:45.871Z" }, + { url = "https://files.pythonhosted.org/packages/76/7e/bc8911719f7084f72fd545f647601ea3532363927f807d296a8c88a62c0d/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db", size = 206811, upload-time = "2026-03-15T18:51:47.308Z" }, + { url = "https://files.pythonhosted.org/packages/e2/40/c430b969d41dda0c465aa36cc7c2c068afb67177bef50905ac371b28ccc7/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8", size = 193706, upload-time = "2026-03-15T18:51:48.849Z" }, + { url = "https://files.pythonhosted.org/packages/48/15/e35e0590af254f7df984de1323640ef375df5761f615b6225ba8deb9799a/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815", size = 202706, upload-time = "2026-03-15T18:51:50.257Z" }, + { url = "https://files.pythonhosted.org/packages/5e/bd/f736f7b9cc5e93a18b794a50346bb16fbfd6b37f99e8f306f7951d27c17c/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a", size = 202497, upload-time = "2026-03-15T18:51:52.012Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ba/2cc9e3e7dfdf7760a6ed8da7446d22536f3d0ce114ac63dee2a5a3599e62/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43", size = 193511, upload-time = "2026-03-15T18:51:53.723Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cb/5be49b5f776e5613be07298c80e1b02a2d900f7a7de807230595c85a8b2e/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0", size = 220133, upload-time = "2026-03-15T18:51:55.333Z" }, + { url = "https://files.pythonhosted.org/packages/83/43/99f1b5dad345accb322c80c7821071554f791a95ee50c1c90041c157ae99/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1", size = 203035, upload-time = "2026-03-15T18:51:56.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/9a/62c2cb6a531483b55dddff1a68b3d891a8b498f3ca555fbcf2978e804d9d/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f", size = 216321, upload-time = "2026-03-15T18:51:58.17Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/94a010ff81e3aec7c293eb82c28f930918e517bc144c9906a060844462eb/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815", size = 208973, upload-time = "2026-03-15T18:51:59.998Z" }, + { url = "https://files.pythonhosted.org/packages/2a/57/4ecff6d4ec8585342f0c71bc03efaa99cb7468f7c91a57b105bcd561cea8/charset_normalizer-3.4.6-cp314-cp314-win32.whl", hash = "sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d", size = 144610, upload-time = "2026-03-15T18:52:02.213Z" }, + { url = "https://files.pythonhosted.org/packages/80/94/8434a02d9d7f168c25767c64671fead8d599744a05d6a6c877144c754246/charset_normalizer-3.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f", size = 154962, upload-time = "2026-03-15T18:52:03.658Z" }, + { url = "https://files.pythonhosted.org/packages/46/4c/48f2cdbfd923026503dfd67ccea45c94fd8fe988d9056b468579c66ed62b/charset_normalizer-3.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e", size = 143595, upload-time = "2026-03-15T18:52:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/8878be7569f87b14f1d52032946131bcb6ebbd8af3e20446bc04053dc3f1/charset_normalizer-3.4.6-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866", size = 314828, upload-time = "2026-03-15T18:52:06.831Z" }, + { url = "https://files.pythonhosted.org/packages/06/b6/fae511ca98aac69ecc35cde828b0a3d146325dd03d99655ad38fc2cc3293/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc", size = 208138, upload-time = "2026-03-15T18:52:08.239Z" }, + { url = "https://files.pythonhosted.org/packages/54/57/64caf6e1bf07274a1e0b7c160a55ee9e8c9ec32c46846ce59b9c333f7008/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e", size = 224679, upload-time = "2026-03-15T18:52:10.043Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cb/9ff5a25b9273ef160861b41f6937f86fae18b0792fe0a8e75e06acb08f1d/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077", size = 223475, upload-time = "2026-03-15T18:52:11.854Z" }, + { url = "https://files.pythonhosted.org/packages/fc/97/440635fc093b8d7347502a377031f9605a1039c958f3cd18dcacffb37743/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f", size = 215230, upload-time = "2026-03-15T18:52:13.325Z" }, + { url = "https://files.pythonhosted.org/packages/cd/24/afff630feb571a13f07c8539fbb502d2ab494019492aaffc78ef41f1d1d0/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e", size = 199045, upload-time = "2026-03-15T18:52:14.752Z" }, + { url = "https://files.pythonhosted.org/packages/e5/17/d1399ecdaf7e0498c327433e7eefdd862b41236a7e484355b8e0e5ebd64b/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484", size = 211658, upload-time = "2026-03-15T18:52:16.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/38/16baa0affb957b3d880e5ac2144caf3f9d7de7bc4a91842e447fbb5e8b67/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7", size = 210769, upload-time = "2026-03-15T18:52:17.782Z" }, + { url = "https://files.pythonhosted.org/packages/05/34/c531bc6ac4c21da9ddfddb3107be2287188b3ea4b53b70fc58f2a77ac8d8/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff", size = 201328, upload-time = "2026-03-15T18:52:19.553Z" }, + { url = "https://files.pythonhosted.org/packages/fa/73/a5a1e9ca5f234519c1953608a03fe109c306b97fdfb25f09182babad51a7/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e", size = 225302, upload-time = "2026-03-15T18:52:21.043Z" }, + { url = "https://files.pythonhosted.org/packages/ba/f6/cd782923d112d296294dea4bcc7af5a7ae0f86ab79f8fefbda5526b6cfc0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659", size = 211127, upload-time = "2026-03-15T18:52:22.491Z" }, + { url = "https://files.pythonhosted.org/packages/0e/c5/0b6898950627af7d6103a449b22320372c24c6feda91aa24e201a478d161/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602", size = 222840, upload-time = "2026-03-15T18:52:24.113Z" }, + { url = "https://files.pythonhosted.org/packages/7d/25/c4bba773bef442cbdc06111d40daa3de5050a676fa26e85090fc54dd12f0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407", size = 216890, upload-time = "2026-03-15T18:52:25.541Z" }, + { url = "https://files.pythonhosted.org/packages/35/1a/05dacadb0978da72ee287b0143097db12f2e7e8d3ffc4647da07a383b0b7/charset_normalizer-3.4.6-cp314-cp314t-win32.whl", hash = "sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579", size = 155379, upload-time = "2026-03-15T18:52:27.05Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7a/d269d834cb3a76291651256f3b9a5945e81d0a49ab9f4a498964e83c0416/charset_normalizer-3.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4", size = 169043, upload-time = "2026-03-15T18:52:28.502Z" }, + { url = "https://files.pythonhosted.org/packages/23/06/28b29fba521a37a8932c6a84192175c34d49f84a6d4773fa63d05f9aff22/charset_normalizer-3.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c", size = 148523, upload-time = "2026-03-15T18:52:29.956Z" }, + { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "open-rtls-replay-demo" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "requests" }, + { name = "websocket-client" }, +] + +[package.metadata] +requires-dist = [ + { name = "requests", specifier = ">=2.33.0" }, + { name = "websocket-client", specifier = ">=1.9.0" }, +] + +[[package]] +name = "requests" +version = "2.33.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", size = 134120, upload-time = "2026-03-30T16:09:15.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] diff --git a/docs/index.md b/docs/index.md index ee3051b..d352b27 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,9 +4,13 @@ Reference documentation for hub architecture, configuration, authentication, and Connector demonstrators live outside the hub runtime under [`connectors/`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors). +Shared connector-agnostic utility scripts live under +[`scripts/`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts). The shared local runtime is documented in [`connectors/local-hub/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/local-hub/README.md). Connector examples currently include [`connectors/gtfs/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/gtfs/README.md) and -[`connectors/opensky/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/opensky/README.md). +[`connectors/opensky/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/opensky/README.md), +plus +[`connectors/replay/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/replay/README.md). diff --git a/connectors/opensky/scripts/check_fence_alignment.py b/scripts/check_fence_alignment.py similarity index 91% rename from connectors/opensky/scripts/check_fence_alignment.py rename to scripts/check_fence_alignment.py index 9e8a43e..22521fe 100644 --- a/connectors/opensky/scripts/check_fence_alignment.py +++ b/scripts/check_fence_alignment.py @@ -7,23 +7,16 @@ import json import math import os -import sys from pathlib import Path from typing import Any import requests -SCRIPT_DIR = Path(__file__).resolve().parent -ROOT_DIR = SCRIPT_DIR.parent -sys.path.insert(0, str(ROOT_DIR)) - -from opensky_support import load_env_file # noqa: E402 - def build_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--env-file", default=os.getenv("OPENSKY_ENV_FILE")) - parser.add_argument("--locations-log", default="connectors/opensky/logs/location_updates.ndjson") + parser.add_argument("--env-file", help="Optional dotenv-style file loaded before resolving HUB_* settings") + parser.add_argument("--locations-log", default="logs/location_updates.ndjson") parser.add_argument("--http-url", default=os.getenv("HUB_HTTP_URL")) parser.add_argument("--token", default=os.getenv("HUB_TOKEN")) return parser @@ -32,16 +25,34 @@ def build_argument_parser() -> argparse.ArgumentParser: def main() -> int: args = build_argument_parser().parse_args() load_env_file(args.env_file) + http_url = args.http_url or os.getenv("HUB_HTTP_URL") if not http_url: raise SystemExit("HUB_HTTP_URL or --http-url is required") + fences = fetch_fences(http_url, args.token or os.getenv("HUB_TOKEN")) normalized = [item for item in (normalize_fence(fence) for fence in fences) if item is not None] locations = load_locations(Path(args.locations_log)) + print(json.dumps(summarize_proximity(locations, normalized), indent=2)) return 0 +def load_env_file(path: str | None) -> None: + if not path: + return + input_path = Path(path) + if not input_path.exists(): + return + with input_path.open("r", encoding="utf-8") as handle: + for line in handle: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + os.environ.setdefault(key.strip(), value.strip()) + + def fetch_fences(http_url: str, token: str | None) -> list[dict[str, Any]]: headers = {"Accept": "application/json"} if token: diff --git a/connectors/gtfs/scripts/log_collision_events.py b/scripts/log_collision_events.py similarity index 55% rename from connectors/gtfs/scripts/log_collision_events.py rename to scripts/log_collision_events.py index 9009c92..768a97c 100644 --- a/connectors/gtfs/scripts/log_collision_events.py +++ b/scripts/log_collision_events.py @@ -11,21 +11,26 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", default="connectors/gtfs/logs/collision_events.ndjson") - parser.add_argument("--env-file", default="connectors/gtfs/.env.local") + parser.add_argument("--output", default="logs/collision_events.ndjson") + parser.add_argument("--env-file") + parser.add_argument("--ws-url") + parser.add_argument("--token") args = parser.parse_args() - logger = Path(__file__).with_name("ws_ndjson_logger.py") command = [ sys.executable, - str(logger), + str(Path(__file__).with_name("ws_ndjson_logger.py")), "--topic", "collision_events", "--output", args.output, - "--env-file", - args.env_file, ] + if args.env_file: + command.extend(["--env-file", args.env_file]) + if args.ws_url: + command.extend(["--ws-url", args.ws_url]) + if args.token: + command.extend(["--token", args.token]) try: return subprocess.call(command) except KeyboardInterrupt: diff --git a/connectors/gtfs/scripts/log_fence_events.py b/scripts/log_fence_events.py similarity index 55% rename from connectors/gtfs/scripts/log_fence_events.py rename to scripts/log_fence_events.py index 61635c0..59cfb44 100644 --- a/connectors/gtfs/scripts/log_fence_events.py +++ b/scripts/log_fence_events.py @@ -11,21 +11,26 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", default="connectors/gtfs/logs/fence_events.ndjson") - parser.add_argument("--env-file", default="connectors/gtfs/.env.local") + parser.add_argument("--output", default="logs/fence_events.ndjson") + parser.add_argument("--env-file") + parser.add_argument("--ws-url") + parser.add_argument("--token") args = parser.parse_args() - logger = Path(__file__).with_name("ws_ndjson_logger.py") command = [ sys.executable, - str(logger), + str(Path(__file__).with_name("ws_ndjson_logger.py")), "--topic", "fence_events", "--output", args.output, - "--env-file", - args.env_file, ] + if args.env_file: + command.extend(["--env-file", args.env_file]) + if args.ws_url: + command.extend(["--ws-url", args.ws_url]) + if args.token: + command.extend(["--token", args.token]) try: return subprocess.call(command) except KeyboardInterrupt: diff --git a/connectors/gtfs/scripts/log_locations.py b/scripts/log_locations.py similarity index 55% rename from connectors/gtfs/scripts/log_locations.py rename to scripts/log_locations.py index bdf7f14..0f72375 100644 --- a/connectors/gtfs/scripts/log_locations.py +++ b/scripts/log_locations.py @@ -11,21 +11,26 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", default="connectors/gtfs/logs/location_updates.ndjson") - parser.add_argument("--env-file", default="connectors/gtfs/.env.local") + parser.add_argument("--output", default="logs/location_updates.ndjson") + parser.add_argument("--env-file") + parser.add_argument("--ws-url") + parser.add_argument("--token") args = parser.parse_args() - logger = Path(__file__).with_name("ws_ndjson_logger.py") command = [ sys.executable, - str(logger), + str(Path(__file__).with_name("ws_ndjson_logger.py")), "--topic", "location_updates", "--output", args.output, - "--env-file", - args.env_file, ] + if args.env_file: + command.extend(["--env-file", args.env_file]) + if args.ws_url: + command.extend(["--ws-url", args.ws_url]) + if args.token: + command.extend(["--token", args.token]) try: return subprocess.call(command) except KeyboardInterrupt: diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml new file mode 100644 index 0000000..da66176 --- /dev/null +++ b/scripts/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "open-rtls-hub-scripts" +version = "0.1.0" +description = "General-purpose Python utility scripts for Open RTLS Hub demos" +requires-python = ">=3.12" +dependencies = [ + "requests>=2.33.0", + "websocket-client>=1.9.0", +] + +[tool.uv] +package = false diff --git a/scripts/uv.lock b/scripts/uv.lock new file mode 100644 index 0000000..804e499 --- /dev/null +++ b/scripts/uv.lock @@ -0,0 +1,142 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "certifi" +version = "2026.2.25" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/60/e3bec1881450851b087e301bedc3daa9377a4d45f1c26aa90b0b235e38aa/charset_normalizer-3.4.6.tar.gz", hash = "sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6", size = 143363, upload-time = "2026-03-15T18:53:25.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/62/c0815c992c9545347aeea7859b50dc9044d147e2e7278329c6e02ac9a616/charset_normalizer-3.4.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab", size = 295154, upload-time = "2026-03-15T18:50:50.88Z" }, + { url = "https://files.pythonhosted.org/packages/a8/37/bdca6613c2e3c58c7421891d80cc3efa1d32e882f7c4a7ee6039c3fc951a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21", size = 199191, upload-time = "2026-03-15T18:50:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/6c/92/9934d1bbd69f7f398b38c5dae1cbf9cc672e7c34a4adf7b17c0a9c17d15d/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2", size = 218674, upload-time = "2026-03-15T18:50:54.102Z" }, + { url = "https://files.pythonhosted.org/packages/af/90/25f6ab406659286be929fd89ab0e78e38aa183fc374e03aa3c12d730af8a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff", size = 215259, upload-time = "2026-03-15T18:50:55.616Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ef/79a463eb0fff7f96afa04c1d4c51f8fc85426f918db467854bfb6a569ce3/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5", size = 207276, upload-time = "2026-03-15T18:50:57.054Z" }, + { url = "https://files.pythonhosted.org/packages/f7/72/d0426afec4b71dc159fa6b4e68f868cd5a3ecd918fec5813a15d292a7d10/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0", size = 195161, upload-time = "2026-03-15T18:50:58.686Z" }, + { url = "https://files.pythonhosted.org/packages/bf/18/c82b06a68bfcb6ce55e508225d210c7e6a4ea122bfc0748892f3dc4e8e11/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a", size = 203452, upload-time = "2026-03-15T18:51:00.196Z" }, + { url = "https://files.pythonhosted.org/packages/44/d6/0c25979b92f8adafdbb946160348d8d44aa60ce99afdc27df524379875cb/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2", size = 202272, upload-time = "2026-03-15T18:51:01.703Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/7fea3e8fe84136bebbac715dd1221cc25c173c57a699c030ab9b8900cbb7/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5", size = 195622, upload-time = "2026-03-15T18:51:03.526Z" }, + { url = "https://files.pythonhosted.org/packages/57/8a/d6f7fd5cb96c58ef2f681424fbca01264461336d2a7fc875e4446b1f1346/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6", size = 220056, upload-time = "2026-03-15T18:51:05.269Z" }, + { url = "https://files.pythonhosted.org/packages/16/50/478cdda782c8c9c3fb5da3cc72dd7f331f031e7f1363a893cdd6ca0f8de0/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d", size = 203751, upload-time = "2026-03-15T18:51:06.858Z" }, + { url = "https://files.pythonhosted.org/packages/75/fc/cc2fcac943939c8e4d8791abfa139f685e5150cae9f94b60f12520feaa9b/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2", size = 216563, upload-time = "2026-03-15T18:51:08.564Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b7/a4add1d9a5f68f3d037261aecca83abdb0ab15960a3591d340e829b37298/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923", size = 209265, upload-time = "2026-03-15T18:51:10.312Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/c094561b5d64a24277707698e54b7f67bd17a4f857bbfbb1072bba07c8bf/charset_normalizer-3.4.6-cp312-cp312-win32.whl", hash = "sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4", size = 144229, upload-time = "2026-03-15T18:51:11.694Z" }, + { url = "https://files.pythonhosted.org/packages/ab/20/0567efb3a8fd481b8f34f739ebddc098ed062a59fed41a8d193a61939e8f/charset_normalizer-3.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb", size = 154277, upload-time = "2026-03-15T18:51:13.004Z" }, + { url = "https://files.pythonhosted.org/packages/15/57/28d79b44b51933119e21f65479d0864a8d5893e494cf5daab15df0247c17/charset_normalizer-3.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4", size = 142817, upload-time = "2026-03-15T18:51:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1d/4fdabeef4e231153b6ed7567602f3b68265ec4e5b76d6024cf647d43d981/charset_normalizer-3.4.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f", size = 294823, upload-time = "2026-03-15T18:51:15.755Z" }, + { url = "https://files.pythonhosted.org/packages/47/7b/20e809b89c69d37be748d98e84dce6820bf663cf19cf6b942c951a3e8f41/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843", size = 198527, upload-time = "2026-03-15T18:51:17.177Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/4f8d27527d59c039dce6f7622593cdcd3d70a8504d87d09eb11e9fdc6062/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf", size = 218388, upload-time = "2026-03-15T18:51:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/f6/9b/4770ccb3e491a9bacf1c46cc8b812214fe367c86a96353ccc6daf87b01ec/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8", size = 214563, upload-time = "2026-03-15T18:51:20.374Z" }, + { url = "https://files.pythonhosted.org/packages/2b/58/a199d245894b12db0b957d627516c78e055adc3a0d978bc7f65ddaf7c399/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9", size = 206587, upload-time = "2026-03-15T18:51:21.807Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/3def227f1ec56f5c69dfc8392b8bd63b11a18ca8178d9211d7cc5e5e4f27/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88", size = 194724, upload-time = "2026-03-15T18:51:23.508Z" }, + { url = "https://files.pythonhosted.org/packages/58/ab/9318352e220c05efd31c2779a23b50969dc94b985a2efa643ed9077bfca5/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84", size = 202956, upload-time = "2026-03-15T18:51:25.239Z" }, + { url = "https://files.pythonhosted.org/packages/75/13/f3550a3ac25b70f87ac98c40d3199a8503676c2f1620efbf8d42095cfc40/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd", size = 201923, upload-time = "2026-03-15T18:51:26.682Z" }, + { url = "https://files.pythonhosted.org/packages/1b/db/c5c643b912740b45e8eec21de1bbab8e7fc085944d37e1e709d3dcd9d72f/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c", size = 195366, upload-time = "2026-03-15T18:51:28.129Z" }, + { url = "https://files.pythonhosted.org/packages/5a/67/3b1c62744f9b2448443e0eb160d8b001c849ec3fef591e012eda6484787c/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194", size = 219752, upload-time = "2026-03-15T18:51:29.556Z" }, + { url = "https://files.pythonhosted.org/packages/f6/98/32ffbaf7f0366ffb0445930b87d103f6b406bc2c271563644bde8a2b1093/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc", size = 203296, upload-time = "2026-03-15T18:51:30.921Z" }, + { url = "https://files.pythonhosted.org/packages/41/12/5d308c1bbe60cabb0c5ef511574a647067e2a1f631bc8634fcafaccd8293/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f", size = 215956, upload-time = "2026-03-15T18:51:32.399Z" }, + { url = "https://files.pythonhosted.org/packages/53/e9/5f85f6c5e20669dbe56b165c67b0260547dea97dba7e187938833d791687/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2", size = 208652, upload-time = "2026-03-15T18:51:34.214Z" }, + { url = "https://files.pythonhosted.org/packages/f1/11/897052ea6af56df3eef3ca94edafee410ca699ca0c7b87960ad19932c55e/charset_normalizer-3.4.6-cp313-cp313-win32.whl", hash = "sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d", size = 143940, upload-time = "2026-03-15T18:51:36.15Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5c/724b6b363603e419829f561c854b87ed7c7e31231a7908708ac086cdf3e2/charset_normalizer-3.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389", size = 154101, upload-time = "2026-03-15T18:51:37.876Z" }, + { url = "https://files.pythonhosted.org/packages/01/a5/7abf15b4c0968e47020f9ca0935fb3274deb87cb288cd187cad92e8cdffd/charset_normalizer-3.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f", size = 143109, upload-time = "2026-03-15T18:51:39.565Z" }, + { url = "https://files.pythonhosted.org/packages/25/6f/ffe1e1259f384594063ea1869bfb6be5cdb8bc81020fc36c3636bc8302a1/charset_normalizer-3.4.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8", size = 294458, upload-time = "2026-03-15T18:51:41.134Z" }, + { url = "https://files.pythonhosted.org/packages/56/60/09bb6c13a8c1016c2ed5c6a6488e4ffef506461aa5161662bd7636936fb1/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421", size = 199277, upload-time = "2026-03-15T18:51:42.953Z" }, + { url = "https://files.pythonhosted.org/packages/00/50/dcfbb72a5138bbefdc3332e8d81a23494bf67998b4b100703fd15fa52d81/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2", size = 218758, upload-time = "2026-03-15T18:51:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/03/b3/d79a9a191bb75f5aa81f3aaaa387ef29ce7cb7a9e5074ba8ea095cc073c2/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30", size = 215299, upload-time = "2026-03-15T18:51:45.871Z" }, + { url = "https://files.pythonhosted.org/packages/76/7e/bc8911719f7084f72fd545f647601ea3532363927f807d296a8c88a62c0d/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db", size = 206811, upload-time = "2026-03-15T18:51:47.308Z" }, + { url = "https://files.pythonhosted.org/packages/e2/40/c430b969d41dda0c465aa36cc7c2c068afb67177bef50905ac371b28ccc7/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8", size = 193706, upload-time = "2026-03-15T18:51:48.849Z" }, + { url = "https://files.pythonhosted.org/packages/48/15/e35e0590af254f7df984de1323640ef375df5761f615b6225ba8deb9799a/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815", size = 202706, upload-time = "2026-03-15T18:51:50.257Z" }, + { url = "https://files.pythonhosted.org/packages/5e/bd/f736f7b9cc5e93a18b794a50346bb16fbfd6b37f99e8f306f7951d27c17c/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a", size = 202497, upload-time = "2026-03-15T18:51:52.012Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ba/2cc9e3e7dfdf7760a6ed8da7446d22536f3d0ce114ac63dee2a5a3599e62/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43", size = 193511, upload-time = "2026-03-15T18:51:53.723Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cb/5be49b5f776e5613be07298c80e1b02a2d900f7a7de807230595c85a8b2e/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0", size = 220133, upload-time = "2026-03-15T18:51:55.333Z" }, + { url = "https://files.pythonhosted.org/packages/83/43/99f1b5dad345accb322c80c7821071554f791a95ee50c1c90041c157ae99/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1", size = 203035, upload-time = "2026-03-15T18:51:56.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/9a/62c2cb6a531483b55dddff1a68b3d891a8b498f3ca555fbcf2978e804d9d/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f", size = 216321, upload-time = "2026-03-15T18:51:58.17Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/94a010ff81e3aec7c293eb82c28f930918e517bc144c9906a060844462eb/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815", size = 208973, upload-time = "2026-03-15T18:51:59.998Z" }, + { url = "https://files.pythonhosted.org/packages/2a/57/4ecff6d4ec8585342f0c71bc03efaa99cb7468f7c91a57b105bcd561cea8/charset_normalizer-3.4.6-cp314-cp314-win32.whl", hash = "sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d", size = 144610, upload-time = "2026-03-15T18:52:02.213Z" }, + { url = "https://files.pythonhosted.org/packages/80/94/8434a02d9d7f168c25767c64671fead8d599744a05d6a6c877144c754246/charset_normalizer-3.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f", size = 154962, upload-time = "2026-03-15T18:52:03.658Z" }, + { url = "https://files.pythonhosted.org/packages/46/4c/48f2cdbfd923026503dfd67ccea45c94fd8fe988d9056b468579c66ed62b/charset_normalizer-3.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e", size = 143595, upload-time = "2026-03-15T18:52:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/31/93/8878be7569f87b14f1d52032946131bcb6ebbd8af3e20446bc04053dc3f1/charset_normalizer-3.4.6-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866", size = 314828, upload-time = "2026-03-15T18:52:06.831Z" }, + { url = "https://files.pythonhosted.org/packages/06/b6/fae511ca98aac69ecc35cde828b0a3d146325dd03d99655ad38fc2cc3293/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc", size = 208138, upload-time = "2026-03-15T18:52:08.239Z" }, + { url = "https://files.pythonhosted.org/packages/54/57/64caf6e1bf07274a1e0b7c160a55ee9e8c9ec32c46846ce59b9c333f7008/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e", size = 224679, upload-time = "2026-03-15T18:52:10.043Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cb/9ff5a25b9273ef160861b41f6937f86fae18b0792fe0a8e75e06acb08f1d/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077", size = 223475, upload-time = "2026-03-15T18:52:11.854Z" }, + { url = "https://files.pythonhosted.org/packages/fc/97/440635fc093b8d7347502a377031f9605a1039c958f3cd18dcacffb37743/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f", size = 215230, upload-time = "2026-03-15T18:52:13.325Z" }, + { url = "https://files.pythonhosted.org/packages/cd/24/afff630feb571a13f07c8539fbb502d2ab494019492aaffc78ef41f1d1d0/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e", size = 199045, upload-time = "2026-03-15T18:52:14.752Z" }, + { url = "https://files.pythonhosted.org/packages/e5/17/d1399ecdaf7e0498c327433e7eefdd862b41236a7e484355b8e0e5ebd64b/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484", size = 211658, upload-time = "2026-03-15T18:52:16.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/38/16baa0affb957b3d880e5ac2144caf3f9d7de7bc4a91842e447fbb5e8b67/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7", size = 210769, upload-time = "2026-03-15T18:52:17.782Z" }, + { url = "https://files.pythonhosted.org/packages/05/34/c531bc6ac4c21da9ddfddb3107be2287188b3ea4b53b70fc58f2a77ac8d8/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff", size = 201328, upload-time = "2026-03-15T18:52:19.553Z" }, + { url = "https://files.pythonhosted.org/packages/fa/73/a5a1e9ca5f234519c1953608a03fe109c306b97fdfb25f09182babad51a7/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e", size = 225302, upload-time = "2026-03-15T18:52:21.043Z" }, + { url = "https://files.pythonhosted.org/packages/ba/f6/cd782923d112d296294dea4bcc7af5a7ae0f86ab79f8fefbda5526b6cfc0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659", size = 211127, upload-time = "2026-03-15T18:52:22.491Z" }, + { url = "https://files.pythonhosted.org/packages/0e/c5/0b6898950627af7d6103a449b22320372c24c6feda91aa24e201a478d161/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602", size = 222840, upload-time = "2026-03-15T18:52:24.113Z" }, + { url = "https://files.pythonhosted.org/packages/7d/25/c4bba773bef442cbdc06111d40daa3de5050a676fa26e85090fc54dd12f0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407", size = 216890, upload-time = "2026-03-15T18:52:25.541Z" }, + { url = "https://files.pythonhosted.org/packages/35/1a/05dacadb0978da72ee287b0143097db12f2e7e8d3ffc4647da07a383b0b7/charset_normalizer-3.4.6-cp314-cp314t-win32.whl", hash = "sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579", size = 155379, upload-time = "2026-03-15T18:52:27.05Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7a/d269d834cb3a76291651256f3b9a5945e81d0a49ab9f4a498964e83c0416/charset_normalizer-3.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4", size = 169043, upload-time = "2026-03-15T18:52:28.502Z" }, + { url = "https://files.pythonhosted.org/packages/23/06/28b29fba521a37a8932c6a84192175c34d49f84a6d4773fa63d05f9aff22/charset_normalizer-3.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c", size = 148523, upload-time = "2026-03-15T18:52:29.956Z" }, + { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "open-rtls-hub-scripts" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "requests" }, + { name = "websocket-client" }, +] + +[package.metadata] +requires-dist = [ + { name = "requests", specifier = ">=2.33.0" }, + { name = "websocket-client", specifier = ">=1.9.0" }, +] + +[[package]] +name = "requests" +version = "2.33.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", size = 134120, upload-time = "2026-03-30T16:09:15.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] diff --git a/connectors/gtfs/scripts/ws_ndjson_logger.py b/scripts/ws_ndjson_logger.py similarity index 71% rename from connectors/gtfs/scripts/ws_ndjson_logger.py rename to scripts/ws_ndjson_logger.py index bf099f3..fe6a21b 100644 --- a/connectors/gtfs/scripts/ws_ndjson_logger.py +++ b/scripts/ws_ndjson_logger.py @@ -7,30 +7,29 @@ import json import os import signal -import sys import time from datetime import datetime, timezone from pathlib import Path import websocket -SCRIPT_DIR = Path(__file__).resolve().parent -ROOT_DIR = SCRIPT_DIR.parent -sys.path.insert(0, str(ROOT_DIR)) - -from gtfs_support import load_env_file # noqa: E402 - RUNNING = True def build_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--topic", required=True) - parser.add_argument("--output", required=True) - parser.add_argument("--env-file", default=os.getenv("GTFS_ENV_FILE")) - parser.add_argument("--ws-url", default=os.getenv("HUB_WS_URL")) - parser.add_argument("--token", default=os.getenv("HUB_TOKEN")) + parser.add_argument("--topic", required=True, help="Hub WebSocket topic to subscribe to") + parser.add_argument("--output", required=True, help="NDJSON file to append received wrapper messages to") + parser.add_argument("--env-file", help="Optional dotenv-style file loaded before resolving HUB_* settings") + parser.add_argument("--ws-url", default=os.getenv("HUB_WS_URL"), help="Hub WebSocket URL") + parser.add_argument("--token", default=os.getenv("HUB_TOKEN"), help="Optional bearer token for subscribe params") + parser.add_argument( + "--reconnect-delay-seconds", + type=float, + default=2.0, + help="Delay before reconnecting after a receive failure", + ) return parser @@ -76,13 +75,28 @@ def main() -> int: except KeyboardInterrupt: break except Exception: - time.sleep(2.0) + time.sleep(max(args.reconnect_delay_seconds, 0.0)) finally: if connection is not None: connection.close() return 0 +def load_env_file(path: str | None) -> None: + if not path: + return + input_path = Path(path) + if not input_path.exists(): + return + with input_path.open("r", encoding="utf-8") as handle: + for line in handle: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + os.environ.setdefault(key.strip(), value.strip()) + + def handle_stop(_signum: int, _frame: object) -> None: global RUNNING RUNNING = False From 701120909201eb9a2b93bd12f9ccf054611dbf18 Mon Sep 17 00:00:00 2001 From: Jilles van Gurp Date: Thu, 2 Apr 2026 09:39:18 +0200 Subject: [PATCH 2/4] Fix repository docs links --- README.md | 36 ++++++++++++++++++------------------ connectors/README.md | 10 +++++----- connectors/gtfs/README.md | 2 +- connectors/replay/README.md | 2 +- docs/auth.md | 2 +- docs/configuration.md | 2 +- docs/index.md | 12 ++++++------ engineering/testing.md | 6 +++--- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index d6f1477..42dd7aa 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Open RTLS Hub is an OpenAPI-first Go implementation of an OMLOX-ready location hub. It provides OMLOX `/v2` REST resources, OMLOX companion MQTT and WebSocket surfaces, and hub-mediated RPC control-plane support for location-driven integrations. -The hub is vendor-neutral and environment-driven. It runs with Postgres, MQTT, and JWT-based access control, and it follows a contract-first workflow with the normative REST contract in [specifications/openapi/omlox-hub.v0.yaml](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/specifications/openapi/omlox-hub.v0.yaml). +The hub is vendor-neutral and environment-driven. It runs with Postgres, MQTT, and JWT-based access control, and it follows a contract-first workflow with the normative REST contract in [specifications/openapi/omlox-hub.v0.yaml](specifications/openapi/omlox-hub.v0.yaml). Key capabilities: - OMLOX `/v2` REST resources and ingestion endpoints @@ -14,7 +14,7 @@ Key capabilities: - Dockerized local runtime for Postgres, Mosquitto, Dex, and the hub - `just` workflows for bootstrap, code generation, validation, and compose operations - Unit tests and Testcontainers-based integration coverage -- Connector demonstrators under [`connectors/`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors) +- Connector demonstrators under [`connectors/`](connectors) ## Omlox @@ -54,25 +54,25 @@ Notes: - `just compose-logs` tails compose services ## Software Docs -- [docs/index.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/docs/index.md) -- [docs/architecture.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/docs/architecture.md) -- [docs/configuration.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/docs/configuration.md) -- [docs/auth.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/docs/auth.md) -- [docs/rpc.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/docs/rpc.md) +- [docs/index.md](docs/index.md) +- [docs/architecture.md](docs/architecture.md) +- [docs/configuration.md](docs/configuration.md) +- [docs/auth.md](docs/auth.md) +- [docs/rpc.md](docs/rpc.md) ## Connector Demonstrators -- [connectors/README.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/README.md) -- [connectors/local-hub/README.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/local-hub/README.md) -- [connectors/gtfs/README.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/gtfs/README.md) -- [connectors/opensky/README.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/opensky/README.md) +- [connectors/README.md](connectors/README.md) +- [connectors/local-hub/README.md](connectors/local-hub/README.md) +- [connectors/gtfs/README.md](connectors/gtfs/README.md) +- [connectors/opensky/README.md](connectors/opensky/README.md) ## Utility Scripts -- [scripts/log_locations.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_locations.py) -- [scripts/log_fence_events.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_fence_events.py) -- [scripts/log_collision_events.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_collision_events.py) -- [scripts/check_fence_alignment.py](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/check_fence_alignment.py) +- [scripts/log_locations.py](scripts/log_locations.py) +- [scripts/log_fence_events.py](scripts/log_fence_events.py) +- [scripts/log_collision_events.py](scripts/log_collision_events.py) +- [scripts/check_fence_alignment.py](scripts/check_fence_alignment.py) ## Engineering Docs -- [engineering/index.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/engineering/index.md) -- [engineering/testing.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/engineering/testing.md) -- [engineering/openapi-governance.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/engineering/openapi-governance.md) +- [engineering/index.md](engineering/index.md) +- [engineering/testing.md](engineering/testing.md) +- [engineering/openapi-governance.md](engineering/openapi-governance.md) diff --git a/connectors/README.md b/connectors/README.md index 87bf4d3..44e9a18 100644 --- a/connectors/README.md +++ b/connectors/README.md @@ -10,11 +10,11 @@ Connector projects in this repository should: - prefer the hub's existing OMLOX interfaces over private integration paths - keep bootstrap utilities and runtime connectors in the same project when they depend on the same upstream metadata -- reuse the shared local hub runtime under [`connectors/local-hub`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/local-hub) when they need a local demo stack +- reuse the shared local hub runtime under [`connectors/local-hub`](local-hub) when they need a local demo stack Available connector demos: -- [`connectors/local-hub`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/local-hub): reusable local hub, Postgres, Dex, and Mosquitto stack -- [`connectors/gtfs`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/gtfs): GTFS-RT vehicle updates and station fence bootstrap -- [`connectors/opensky`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/opensky): OpenSky aircraft positions with airport-sector fences -- [`connectors/replay`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/replay): diagnostic NDJSON trace replay with timestamp correction, acceleration, and interpolation +- [`connectors/local-hub`](local-hub): reusable local hub, Postgres, Dex, and Mosquitto stack +- [`connectors/gtfs`](gtfs): GTFS-RT vehicle updates and station fence bootstrap +- [`connectors/opensky`](opensky): OpenSky aircraft positions with airport-sector fences +- [`connectors/replay`](replay): diagnostic NDJSON trace replay with timestamp correction, acceleration, and interpolation diff --git a/connectors/gtfs/README.md b/connectors/gtfs/README.md index 3600fdb..4ee45fa 100644 --- a/connectors/gtfs/README.md +++ b/connectors/gtfs/README.md @@ -28,7 +28,7 @@ The checked-in defaults target the live Grand Dole network: ## Shared Local Hub This demo uses the shared local runtime in -[`connectors/local-hub`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/local-hub). +[`connectors/local-hub`](../local-hub). Start it with: diff --git a/connectors/replay/README.md b/connectors/replay/README.md index 9454a5e..7b0c0a3 100644 --- a/connectors/replay/README.md +++ b/connectors/replay/README.md @@ -3,7 +3,7 @@ This connector is a diagnostic tool for replaying logged hub `location_updates` NDJSON back into an Open RTLS Hub. It is intended for trace files produced by the shared root logging scripts such as -[`scripts/log_locations.py`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts/log_locations.py). +[`scripts/log_locations.py`](../../scripts/log_locations.py). Replay behavior: diff --git a/docs/auth.md b/docs/auth.md index 1f0bf07..c3bf1e2 100644 --- a/docs/auth.md +++ b/docs/auth.md @@ -172,7 +172,7 @@ If a topic is valid but disabled by configuration, the WebSocket layer returns a ## Dex Development Setup -This repository includes a Dex fixture at [tools/dex/config.yaml](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/tools/dex/config.yaml) and a matching permissions file at [config/auth/permissions.yaml](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/config/auth/permissions.yaml). +This repository includes a Dex fixture at [tools/dex/config.yaml](../tools/dex/config.yaml) and a matching permissions file at [config/auth/permissions.yaml](../config/auth/permissions.yaml). `docker compose` starts Dex on port `5556` and configures the app container to verify Dex-issued tokens with: diff --git a/docs/configuration.md b/docs/configuration.md index 166feda..bb89002 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -93,7 +93,7 @@ Proximity resolution behavior: - `AUTH_ROLES_CLAIM` (JWT claim used for role extraction, default `groups`) - `AUTH_OWNED_RESOURCES_CLAIM` (JWT object claim for owned resource IDs; see `docs/auth.md` for format and usage) -See [docs/auth.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/docs/auth.md) for the full auth model, Dex setup, and permission file format. +See [docs/auth.md](auth.md) for the full auth model, Dex setup, and permission file format. ## RPC Security Defaults diff --git a/docs/index.md b/docs/index.md index d352b27..94daa6f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,14 +3,14 @@ Reference documentation for hub architecture, configuration, authentication, and RPC behavior. Connector demonstrators live outside the hub runtime under -[`connectors/`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors). +[`connectors/`](../connectors). Shared connector-agnostic utility scripts live under -[`scripts/`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/scripts). +[`scripts/`](../scripts). The shared local runtime is documented in -[`connectors/local-hub/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/local-hub/README.md). +[`connectors/local-hub/README.md`](../connectors/local-hub/README.md). Connector examples currently include -[`connectors/gtfs/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/gtfs/README.md) +[`connectors/gtfs/README.md`](../connectors/gtfs/README.md) and -[`connectors/opensky/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/opensky/README.md), +[`connectors/opensky/README.md`](../connectors/opensky/README.md), plus -[`connectors/replay/README.md`](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/connectors/replay/README.md). +[`connectors/replay/README.md`](../connectors/replay/README.md). diff --git a/engineering/testing.md b/engineering/testing.md index 687a9dc..6694fda 100644 --- a/engineering/testing.md +++ b/engineering/testing.md @@ -55,8 +55,8 @@ The lint stack now verifies: - `go vet` - `staticcheck` - `govulncheck` -- `go mod tidy` leaves [go.mod](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/go.mod) and [go.sum](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/go.sum) unchanged -- `just generate` leaves [internal/httpapi/gen/api.gen.go](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/internal/httpapi/gen/api.gen.go) and [internal/storage/postgres/sqlcgen](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/internal/storage/postgres/sqlcgen) unchanged +- `go mod tidy` leaves [go.mod](../go.mod) and [go.sum](../go.sum) unchanged +- `just generate` leaves [internal/httpapi/gen/api.gen.go](../internal/httpapi/gen/api.gen.go) and [internal/storage/postgres/sqlcgen](../internal/storage/postgres/sqlcgen) unchanged ## Integration tests Run integration tests with Docker/Testcontainers: @@ -86,4 +86,4 @@ The integration suite now also includes shared-hub scenario coverage for high-tr - Linux and Docker builds install native PROJ packages and are the expected path for CRS behavior and its test coverage. - GitHub Actions Ubuntu runners also need native PROJ packages before `just lint`, `just check`, or `just build`; the CI workflow installs `pkg-config`, `libproj-dev`, and `proj-data` explicitly and caches apt archives to reduce repeated package download cost. - direct `go test` or `go build` runs should export `PKG_CONFIG="$PWD/tools/bin/pkg-config"` if `pkg-config` is not already available globally. -- Auth setup, Dex fixtures, and permission examples are documented in [docs/auth.md](/Users/jillesvangurp/git/open-rtls/open-rtls-hub/docs/auth.md). +- Auth setup, Dex fixtures, and permission examples are documented in [docs/auth.md](../docs/auth.md). From acce2452973a8d954d0f9474c45ba6f0122d12f3 Mon Sep 17 00:00:00 2001 From: Jilles van Gurp Date: Thu, 2 Apr 2026 11:02:43 +0200 Subject: [PATCH 3/4] Make derived location processing asynchronous --- cmd/hub/main.go | 1 + docs/architecture.md | 9 +- docs/configuration.md | 3 + internal/config/config.go | 5 + internal/config/config_test.go | 15 +++ internal/hub/derived_processor.go | 141 ++++++++++++++++++++++++++ internal/hub/service.go | 161 ++++++++++++++++++++++++++++++ internal/hub/service_test.go | 51 ++++++++++ 8 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 internal/hub/derived_processor.go diff --git a/cmd/hub/main.go b/cmd/hub/main.go index 8f85ff6..c183fda 100644 --- a/cmd/hub/main.go +++ b/cmd/hub/main.go @@ -170,6 +170,7 @@ func runWithRuntime(ctx context.Context, rt runtimeDeps) error { LocationTTL: cfg.StateLocationTTL, ProximityTTL: cfg.StateProximityTTL, DedupTTL: cfg.StateDedupTTL, + DerivedLocationBuffer: cfg.DerivedLocationBuffer, MetadataReconcileInterval: cfg.MetadataReconcileInterval, CollisionsEnabled: cfg.CollisionsEnabled, CollisionStateTTL: cfg.CollisionStateTTL, diff --git a/docs/architecture.md b/docs/architecture.md index 022c7c3..77d468c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -27,14 +27,17 @@ ## Event Fan-Out 1. REST, MQTT, or WebSocket ingest enters the shared hub service. -2. The hub validates, normalizes, deduplicates, updates in-memory transient state, and derives follow-on events. -3. The hub emits normalized internal events for locations, proximities, trackable motions, fence events, optional collision events, and metadata changes. -4. MQTT and WebSocket consume that same event stream and publish transport-specific payloads. +2. The hub validates, normalizes, deduplicates, updates in-memory transient state, and emits the native location event on the ingest path. +3. A buffered in-process derived-location worker handles non-critical follow-on work such as alternate-CRS publication, geofence evaluation, and optional collision evaluation. +4. When that derived queue is full, the hub drops derived work for newer locations rather than backpressuring the ingest path. +5. MQTT and WebSocket consume the resulting internal event stream and publish transport-specific payloads. Implications: - ingest logic is shared across REST, MQTT, and WebSocket - MQTT is no longer the only downstream publication path - the internal event seam decouples downstream publication from MQTT-specific topics +- location ingest latency is protected from slower geofence or collision work +- the derived-location queue is the intended insertion point for future filtered or smoothed track processing before fence/collision decisions - hub-issued UUIDs for REST-managed resources, derived fence/collision events, and RPC caller IDs now use UUIDv7 so emitted identifiers are time-sortable - internal hub events carry the persisted `origin_hub_id` so downstream transports preserve source provenance diff --git a/docs/configuration.md b/docs/configuration.md index 166feda..a27c2ff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,6 +20,7 @@ Runtime lifecycle behavior: - `WEBSOCKET_READ_TIMEOUT` (duration, default `1m`) - `WEBSOCKET_PING_INTERVAL` (duration, default `30s`) - `WEBSOCKET_OUTBOUND_BUFFER` (default `32`) +- `DERIVED_LOCATION_BUFFER` (default `1024`) Hub metadata bootstrap behavior: - the hub persists one durable metadata row in Postgres containing the stable `hub_id` and operator-facing label @@ -53,6 +54,8 @@ Stateful ingest behavior: - durable hub metadata is also loaded from Postgres at startup before the service begins accepting traffic - WebSocket delivery uses a per-connection outbound queue capped by `WEBSOCKET_OUTBOUND_BUFFER`; slow subscribers are disconnected instead of backpressuring the ingest path - WebSocket liveness uses server ping frames every `WEBSOCKET_PING_INTERVAL` and considers the connection stale when no inbound message or pong arrives before `WEBSOCKET_READ_TIMEOUT` +- non-critical derived location work such as alternate-CRS publication, geofence evaluation, and collision evaluation is queued behind `DERIVED_LOCATION_BUFFER` +- when the derived queue is full, the hub drops new derived work instead of slowing raw location ingest - the `metadata_changes` WebSocket topic emits lightweight `{id,type,operation,timestamp}` notifications for zone, fence, trackable, and location-provider CRUD or reconcile drift - when `COLLISIONS_ENABLED=true`, the hub evaluates trackable-versus-trackable collisions from the latest WGS84 motion state and keeps short-lived collision pair state in memory for `COLLISION_STATE_TTL` - `COLLISION_COLLIDING_DEBOUNCE` limits repeated `colliding` emissions for already-active pairs diff --git a/internal/config/config.go b/internal/config/config.go index 9e70114..7f161b5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,7 @@ type Config struct { WebSocketReadTimeout time.Duration WebSocketPingInterval time.Duration WebSocketOutboundBuffer int + DerivedLocationBuffer int StateLocationTTL time.Duration StateProximityTTL time.Duration StateDedupTTL time.Duration @@ -82,6 +83,7 @@ func fromLookupEnv(lookup lookupEnvFunc) (Config, error) { WebSocketReadTimeout: durationEnvWithLookup(lookup, "WEBSOCKET_READ_TIMEOUT", time.Minute), WebSocketPingInterval: durationEnvWithLookup(lookup, "WEBSOCKET_PING_INTERVAL", 30*time.Second), WebSocketOutboundBuffer: intEnvWithLookup(lookup, "WEBSOCKET_OUTBOUND_BUFFER", 32), + DerivedLocationBuffer: intEnvWithLookup(lookup, "DERIVED_LOCATION_BUFFER", 1024), StateLocationTTL: durationEnvWithLookup(lookup, "STATE_LOCATION_TTL", 10*time.Minute), StateProximityTTL: durationEnvWithLookup(lookup, "STATE_PROXIMITY_TTL", 5*time.Minute), StateDedupTTL: durationEnvWithLookup(lookup, "STATE_DEDUP_TTL", 2*time.Minute), @@ -144,6 +146,9 @@ func fromLookupEnv(lookup lookupEnvFunc) (Config, error) { if cfg.WebSocketOutboundBuffer <= 0 { return Config{}, fmt.Errorf("WEBSOCKET_OUTBOUND_BUFFER must be > 0") } + if cfg.DerivedLocationBuffer <= 0 { + return Config{}, fmt.Errorf("DERIVED_LOCATION_BUFFER must be > 0") + } if cfg.StateProximityTTL <= 0 { return Config{}, fmt.Errorf("STATE_PROXIMITY_TTL must be > 0") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f0bc2de..4e62236 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -42,6 +42,9 @@ func TestDefaults(t *testing.T) { if cfg.WebSocketWriteTimeout <= 0 || cfg.WebSocketReadTimeout <= 0 || cfg.WebSocketPingInterval <= 0 || cfg.WebSocketOutboundBuffer <= 0 { t.Fatal("expected positive websocket settings") } + if cfg.DerivedLocationBuffer <= 0 { + t.Fatal("expected positive derived location buffer") + } if cfg.WebSocketReadTimeout <= cfg.WebSocketPingInterval { t.Fatal("expected websocket read timeout to exceed ping interval") } @@ -112,6 +115,18 @@ func TestWebSocketReadTimeoutMustExceedPingInterval(t *testing.T) { } } +func TestDerivedLocationBufferMustBePositive(t *testing.T) { + t.Parallel() + + _, err := configFromMap(map[string]string{ + "AUTH_MODE": "none", + "DERIVED_LOCATION_BUFFER": "0", + }) + if err == nil { + t.Fatal("expected validation error") + } +} + func TestValidHubIDLoadsSuccessfully(t *testing.T) { t.Parallel() diff --git a/internal/hub/derived_processor.go b/internal/hub/derived_processor.go new file mode 100644 index 0000000..a61955a --- /dev/null +++ b/internal/hub/derived_processor.go @@ -0,0 +1,141 @@ +package hub + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/formation-res/open-rtls-hub/internal/httpapi/gen" + "go.uber.org/zap" +) + +type derivedLocationWork struct { + Location gen.Location +} + +type derivedLocationSubmitter interface { + Submit(work derivedLocationWork) +} + +type derivedLocationProcessor struct { + service *Service + logger *zap.Logger + queue chan derivedLocationWork + dropped atomic.Uint64 +} + +func startDerivedLocationProcessor(ctx context.Context, service *Service, buffer int) derivedLocationSubmitter { + if service == nil || buffer <= 0 { + return nil + } + processor := &derivedLocationProcessor{ + service: service, + logger: service.logger, + queue: make(chan derivedLocationWork, buffer), + } + go processor.run(ctx) + return processor +} + +func (p *derivedLocationProcessor) Submit(work derivedLocationWork) { + select { + case p.queue <- work: + default: + dropped := p.dropped.Add(1) + if dropped == 1 || dropped%100 == 0 { + p.logger.Warn("derived location queue full; dropping derived work", zap.Uint64("dropped", dropped)) + } + } +} + +func (p *derivedLocationProcessor) run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case work := <-p.queue: + if err := p.service.processDerivedLocation(ctx, work.Location); err != nil { + p.logger.Warn("derived location processing failed", zap.Error(err), zap.String("provider_id", work.Location.ProviderId), zap.String("source", work.Location.Source)) + } + } + } +} + +type derivedLocationView struct { + service *Service + location gen.Location + + localOnce sync.Once + localLoc gen.Location + localOK bool + localErr error + + wgsOnce sync.Once + wgsLoc gen.Location + wgsOK bool + wgsErr error +} + +func newDerivedLocationView(service *Service, location gen.Location) *derivedLocationView { + return &derivedLocationView{service: service, location: location} +} + +func (v *derivedLocationView) NativeScope() EventScope { + return nativeLocationScope(v.location) +} + +func (v *derivedLocationView) WGS84(ctx context.Context) (*gen.Location, bool, error) { + v.wgsOnce.Do(func() { + if locationCRS(v.location) == "EPSG:4326" { + out, err := cloneLocation(v.location) + if err != nil { + v.wgsErr = err + return + } + epsg := "EPSG:4326" + out.Crs = &epsg + v.wgsLoc = out + v.wgsOK = true + return + } + out, err := v.service.locationToWGS84(ctx, v.location) + if err != nil { + v.wgsErr = err + return + } + v.wgsLoc = out + v.wgsOK = true + }) + if !v.wgsOK { + return nil, false, v.wgsErr + } + return &v.wgsLoc, true, nil +} + +func (v *derivedLocationView) Local(ctx context.Context) (*gen.Location, bool, error) { + v.localOnce.Do(func() { + if nativeLocationScope(v.location) == ScopeLocal { + out, err := cloneLocation(v.location) + if err != nil { + v.localErr = err + return + } + localCRS := "local" + out.Crs = &localCRS + v.localLoc = out + v.localOK = true + return + } + out, err := v.service.locationToLocal(ctx, v.location) + if err != nil { + v.localErr = err + return + } + v.localLoc = out + v.localOK = true + }) + if !v.localOK { + return nil, false, v.localErr + } + return &v.localLoc, true, nil +} diff --git a/internal/hub/service.go b/internal/hub/service.go index 55398fb..fad34b9 100644 --- a/internal/hub/service.go +++ b/internal/hub/service.go @@ -31,6 +31,7 @@ type Config struct { LocationTTL time.Duration ProximityTTL time.Duration DedupTTL time.Duration + DerivedLocationBuffer int ProximityResolutionEntryConfidenceMin float64 ProximityResolutionExitGraceDuration time.Duration ProximityResolutionBoundaryGrace float64 @@ -56,6 +57,7 @@ type Service struct { transformCache *transform.Cache metadata *MetadataCache state *ProcessingState + derivedQueue derivedLocationSubmitter } // HTTPError represents an API error that should be rendered with a specific @@ -77,6 +79,9 @@ func (s *Service) Start(ctx context.Context) { return } s.processingState().StartSweeper(ctx, time.Second) + if s.derivedQueue == nil { + s.derivedQueue = startDerivedLocationProcessor(ctx, s, s.cfg.DerivedLocationBuffer) + } if s.metadata == nil || s.queries == nil { return } @@ -105,6 +110,9 @@ func New(logger *zap.Logger, queries sqlcgen.Querier, bus *EventBus, cfg Config) if cfg.MetadataReconcileInterval <= 0 { cfg.MetadataReconcileInterval = 30 * time.Second } + if cfg.DerivedLocationBuffer <= 0 { + cfg.DerivedLocationBuffer = 1024 + } metadata, err := NewMetadataCache(context.Background(), queries) if err != nil { return nil, err @@ -891,6 +899,16 @@ func (s *Service) recordLocation(ctx context.Context, location gen.Location, ttl s.processingState().SetTrackableLocation(latestTrackableLocationKey(trackableID), location, ttl) } } + if s.derivedQueue != nil { + if err := s.publishNativeLocation(ctx, location); err != nil { + s.logger.Warn("location event emit failed", zap.Error(err), zap.String("provider_id", location.ProviderId)) + } + if _, err := s.publishNativeTrackableMotions(ctx, location); err != nil { + s.logger.Warn("trackable motion event emit failed", zap.Error(err), zap.String("provider_id", location.ProviderId)) + } + s.derivedQueue.Submit(derivedLocationWork{Location: location}) + return nil + } if err := s.publishLocation(ctx, location); err != nil { s.logger.Warn("location event emit failed", zap.Error(err), zap.String("provider_id", location.ProviderId)) } @@ -909,6 +927,62 @@ func (s *Service) recordLocation(ctx context.Context, location gen.Location, ttl return nil } +func (s *Service) processDerivedLocation(ctx context.Context, location gen.Location) error { + if s.bus == nil { + return nil + } + view := newDerivedLocationView(s, location) + if err := s.publishFenceEvents(ctx, location); err != nil { + return err + } + switch view.NativeScope() { + case ScopeLocal: + wgs84Location, ok, err := view.WGS84(ctx) + if err == nil && ok { + if err := s.publishLocationScope(ctx, *wgs84Location, ScopeEPSG4326); err != nil { + return err + } + wgs84Motions, err := s.publishTrackableMotionsForLocation(ctx, *wgs84Location, ScopeEPSG4326) + if err != nil { + return err + } + if s.cfg.CollisionsEnabled { + if err := s.publishCollisionEvents(ctx, wgs84Motions); err != nil { + return err + } + } + } + case ScopeEPSG4326: + localLocation, ok, err := view.Local(ctx) + if err == nil && ok { + if err := s.publishLocationScope(ctx, *localLocation, ScopeLocal); err != nil { + return err + } + if _, err := s.publishTrackableMotionsForLocation(ctx, *localLocation, ScopeLocal); err != nil { + return err + } + } + if s.cfg.CollisionsEnabled { + wgs84Motions, err := s.buildTrackableMotionsForLocation(ctx, location) + if err != nil { + return err + } + if err := s.publishCollisionEvents(ctx, wgs84Motions); err != nil { + return err + } + } + } + return nil +} + +func (s *Service) publishNativeLocation(ctx context.Context, location gen.Location) error { + return s.publishLocationScope(ctx, location, nativeLocationScope(location)) +} + +func (s *Service) publishNativeTrackableMotions(ctx context.Context, location gen.Location) ([]gen.TrackableMotion, error) { + return s.publishTrackableMotionsForLocation(ctx, location, nativeLocationScope(location)) +} + func (s *Service) publishLocation(ctx context.Context, location gen.Location) error { if s.bus == nil { return nil @@ -942,6 +1016,22 @@ func (s *Service) publishLocation(ctx context.Context, location gen.Location) er return nil } +func (s *Service) publishLocationScope(_ context.Context, location gen.Location, scope EventScope) error { + if s.bus == nil { + return nil + } + feature, err := locationGeoJSONFeatureCollection(location) + if err != nil { + return err + } + event, err := newEvent(EventLocation, scope, locationTime(location), location.ProviderId, "", "", s.cfg.HubID, LocationEnvelope{Location: location, GeoJSON: feature}) + if err != nil { + return err + } + s.bus.Emit(event) + return nil +} + func (s *Service) publishTrackableMotions(ctx context.Context, location gen.Location) ([]gen.TrackableMotion, error) { if s.bus == nil || location.Trackables == nil { return nil, nil @@ -992,6 +1082,77 @@ func (s *Service) publishTrackableMotions(ctx context.Context, location gen.Loca return wgsMotions, nil } +func (s *Service) publishTrackableMotionsForLocation(ctx context.Context, location gen.Location, scope EventScope) ([]gen.TrackableMotion, error) { + motions, err := s.buildTrackableMotionsForLocation(ctx, location) + if err != nil { + return nil, err + } + return motions, s.publishTrackableMotionEvents(location.ProviderId, scope, motions) +} + +func (s *Service) buildTrackableMotionsForLocation(ctx context.Context, location gen.Location) ([]gen.TrackableMotion, error) { + if s.bus == nil || location.Trackables == nil { + return nil, nil + } + motions := make([]gen.TrackableMotion, 0, len(*location.Trackables)) + for _, id := range *location.Trackables { + baseMotion, err := s.trackableMotionBase(ctx, id) + if err != nil { + return nil, err + } + baseMotion.Location = location + motions = append(motions, baseMotion) + } + return motions, nil +} + +func (s *Service) publishTrackableMotionEvents(providerID string, scope EventScope, motions []gen.TrackableMotion) error { + if s.bus == nil { + return nil + } + for _, motion := range motions { + event, err := newEvent(EventTrackableMotion, scope, locationTime(motion.Location), providerID, motion.Id, "", s.cfg.HubID, TrackableMotionEnvelope{Motion: motion}) + if err != nil { + return err + } + s.bus.Emit(event) + } + return nil +} + +func (s *Service) trackableMotionBase(ctx context.Context, id string) (gen.TrackableMotion, error) { + baseMotion := gen.TrackableMotion{Id: id} + if cache := s.metadataCache(); cache != nil { + if trackable, ok := cache.TrackableByID(id); ok { + baseMotion.Name = trackable.Name + baseMotion.Geometry = trackable.Geometry + baseMotion.Extrusion = trackable.Extrusion + baseMotion.Properties = trackable.Properties + } + return baseMotion, nil + } + parsed, parseErr := uuid.Parse(id) + if parseErr != nil { + return baseMotion, nil + } + trackable, err := s.GetTrackable(ctx, openapi_types.UUID(parsed)) + if err != nil { + return baseMotion, nil + } + baseMotion.Name = trackable.Name + baseMotion.Geometry = trackable.Geometry + baseMotion.Extrusion = trackable.Extrusion + baseMotion.Properties = trackable.Properties + return baseMotion, nil +} + +func nativeLocationScope(location gen.Location) EventScope { + if locationCRS(location) == "EPSG:4326" { + return ScopeEPSG4326 + } + return ScopeLocal +} + type locationPublicationVariants struct { Local *gen.Location WGS84 *gen.Location diff --git a/internal/hub/service_test.go b/internal/hub/service_test.go index b231add..54b8a82 100644 --- a/internal/hub/service_test.go +++ b/internal/hub/service_test.go @@ -592,6 +592,57 @@ func TestPublishFenceEventsUsesLocationTTLForMembershipState(t *testing.T) { } } +type capturingDerivedSubmitter struct { + works []derivedLocationWork +} + +func (c *capturingDerivedSubmitter) Submit(work derivedLocationWork) { + c.works = append(c.works, work) +} + +func TestRecordLocationWithDerivedQueuePublishesNativeAndQueuesDerivedWork(t *testing.T) { + t.Parallel() + + bus := NewEventBus() + ch, unsubscribe := bus.Subscribe(8) + defer unsubscribe() + queue := &capturingDerivedSubmitter{} + crs := "EPSG:4326" + location := testLocationWithCoordinates(t, &crs, "external-source", [2]float32{8.5, 47.3}) + trackables := []string{"trackable-a"} + location.Trackables = &trackables + + service := &Service{ + bus: bus, + derivedQueue: queue, + cfg: Config{ + LocationTTL: time.Minute, + DedupTTL: time.Minute, + CollisionStateTTL: time.Minute, + CollisionCollidingDebounce: time.Second, + MetadataReconcileInterval: time.Second, + DerivedLocationBuffer: 16, + }, + metadata: &MetadataCache{snapshot: newMetadataSnapshot(nil, nil, nil, nil)}, + state: NewProcessingState(time.Now), + logger: zapTestLogger(t), + } + + if err := service.recordLocation(context.Background(), location, time.Minute); err != nil { + t.Fatalf("recordLocation failed: %v", err) + } + events := collectEvents(ch, 2) + if len(events) != 2 { + t.Fatalf("expected 2 native events, got %d", len(events)) + } + if got := decodeEventLocation(t, eventByScope(t, events, ScopeEPSG4326)); got.Source != location.Source { + t.Fatalf("unexpected native location source: %s", got.Source) + } + if len(queue.works) != 1 { + t.Fatalf("expected one queued derived work item, got %d", len(queue.works)) + } +} + func testPolicy() proximityResolutionPolicy { return proximityResolutionPolicy{ ExitGraceDuration: 15 * time.Second, From da5c58fb5b112f96adde4395c3cb348cf0fe7d89 Mon Sep 17 00:00:00 2001 From: Jilles van Gurp Date: Thu, 2 Apr 2026 12:01:44 +0200 Subject: [PATCH 4/4] Improve replay throughput and async fan-out --- cmd/hub/main.go | 4 +- connectors/replay/.env.example | 2 + connectors/replay/README.md | 23 +++ connectors/replay/connector.py | 49 +++++- docs/architecture.md | 14 +- docs/configuration.md | 14 +- internal/config/config.go | 12 +- internal/config/config_test.go | 27 ++++ internal/hub/derived_processor.go | 48 ++++-- internal/hub/events.go | 14 +- internal/hub/processing_state.go | 15 ++ internal/hub/runtime_stats.go | 64 ++++++++ internal/hub/service.go | 104 ++++++++++--- internal/hub/service_test.go | 44 +++++- internal/ws/hub.go | 243 ++++++++++++++++++++++-------- internal/ws/hub_test.go | 5 +- 16 files changed, 557 insertions(+), 125 deletions(-) create mode 100644 internal/hub/runtime_stats.go diff --git a/cmd/hub/main.go b/cmd/hub/main.go index c183fda..60d6bf6 100644 --- a/cmd/hub/main.go +++ b/cmd/hub/main.go @@ -170,6 +170,7 @@ func runWithRuntime(ctx context.Context, rt runtimeDeps) error { LocationTTL: cfg.StateLocationTTL, ProximityTTL: cfg.StateProximityTTL, DedupTTL: cfg.StateDedupTTL, + NativeLocationBuffer: cfg.NativeLocationBuffer, DerivedLocationBuffer: cfg.DerivedLocationBuffer, MetadataReconcileInterval: cfg.MetadataReconcileInterval, CollisionsEnabled: cfg.CollisionsEnabled, @@ -191,7 +192,7 @@ func runWithRuntime(ctx context.Context, rt runtimeDeps) error { if eventBus != nil { var ch <-chan hub.Event var unsubscribeMQTTPublisher func() - ch, unsubscribeMQTTPublisher = eventBus.Subscribe(128) + ch, unsubscribeMQTTPublisher = eventBus.Subscribe(cfg.EventBusSubscriberBuffer) mqttPublisherDone := runEventPublisher(ctx, logger, ch, rt.eventPublisherHandle(mq)) cleanupMQTTPublisher = func() { unsubscribeMQTTPublisher() @@ -272,6 +273,7 @@ func runWithRuntime(ctx context.Context, rt runtimeDeps) error { cfg.WebSocketReadTimeout, cfg.WebSocketPingInterval, cfg.WebSocketOutboundBuffer, + cfg.EventBusSubscriberBuffer, cfg.CollisionsEnabled, ) r.Get("/v2/ws/socket", wsHub.Handle) diff --git a/connectors/replay/.env.example b/connectors/replay/.env.example index 3204874..1eee0ea 100644 --- a/connectors/replay/.env.example +++ b/connectors/replay/.env.example @@ -5,5 +5,7 @@ HUB_TOKEN= REPLAY_INPUT=connectors/gtfs/logs/location_updates.ndjson REPLAY_ACCELERATION_FACTOR=1.0 REPLAY_INTERPOLATION_RATE_HZ=0.0 +REPLAY_BATCH_WINDOW_MS=25.0 +REPLAY_MAX_BATCH_SIZE=256 LOG_LEVEL=INFO diff --git a/connectors/replay/README.md b/connectors/replay/README.md index 7b0c0a3..19ae86f 100644 --- a/connectors/replay/README.md +++ b/connectors/replay/README.md @@ -12,6 +12,8 @@ Replay behavior: - supports real-time replay or faster playback via `--acceleration-factor` - can emit synthetic straight-line interpolation points per object via `--interpolation-rate-hz` +- batches due replay emissions into fewer WebSocket publishes via + `--batch-window-ms` and `--max-batch-size` - best-effort bootstraps referenced providers and trackables when `HUB_HTTP_URL` is configured @@ -50,6 +52,9 @@ Optional but recommended: - `REPLAY_ACCELERATION_FACTOR`: playback speed multiplier, where `1.0` is real time - `REPLAY_INTERPOLATION_RATE_HZ`: per-object interpolation cadence in Hertz, where `1.0` means once per second and `0.1` means once every 10 seconds +- `REPLAY_BATCH_WINDOW_MS`: coalesce replay events due inside this window into one + WebSocket publish +- `REPLAY_MAX_BATCH_SIZE`: cap the number of locations sent in one replay publish ## Setup @@ -121,6 +126,17 @@ uv run --project connectors/replay python connectors/replay/connector.py \ --interpolation-rate-hz 0.1 ``` +Replay with interpolation and larger WebSocket batches: + +```bash +uv run --project connectors/replay python connectors/replay/connector.py \ + --env-file connectors/replay/.env.local \ + --input connectors/opensky/logs/location_updates.ndjson \ + --interpolation-rate-hz 10.0 \ + --batch-window-ms 25 \ + --max-batch-size 256 +``` + ## Input Format The connector expects the NDJSON shape produced by the shared logging scripts: @@ -164,6 +180,13 @@ Synthetic locations: - preserve the original source timestamp in `properties.replay_original_timestamp_generated` +Batching behavior: + +- the connector waits for the first due replay timestamp in a batch +- it then publishes all subsequent events scheduled within `batch-window-ms` +- batching reduces per-frame overhead but does not change the replay timestamps + carried in the payload + ## Limitations - interpolation is linear in WGS84 longitude and latitude; it is intended for diff --git a/connectors/replay/connector.py b/connectors/replay/connector.py index f20829f..e9d679e 100644 --- a/connectors/replay/connector.py +++ b/connectors/replay/connector.py @@ -32,6 +32,18 @@ def build_argument_parser() -> argparse.ArgumentParser: default=float(os.getenv("REPLAY_INTERPOLATION_RATE_HZ", "0.0")), help="Synthetic interpolation cadence per object in Hertz. 1.0 emits once per second.", ) + parser.add_argument( + "--batch-window-ms", + type=float, + default=float(os.getenv("REPLAY_BATCH_WINDOW_MS", "25.0")), + help="Group replay events due within this many milliseconds into one WebSocket publish.", + ) + parser.add_argument( + "--max-batch-size", + type=int, + default=int(os.getenv("REPLAY_MAX_BATCH_SIZE", "256")), + help="Maximum number of locations to send in one replay publish.", + ) return parser @@ -45,6 +57,10 @@ def main() -> int: raise SystemExit("--acceleration-factor must be greater than 0") if args.interpolation_rate_hz < 0: raise SystemExit("--interpolation-rate-hz must be greater than or equal to 0") + if args.batch_window_ms < 0: + raise SystemExit("--batch-window-ms must be greater than or equal to 0") + if args.max_batch_size <= 0: + raise SystemExit("--max-batch-size must be greater than 0") logging.basicConfig( level=os.getenv("LOG_LEVEL", "INFO").upper(), @@ -79,14 +95,15 @@ def main() -> int: start_monotonic = time.monotonic() try: - for event in replay_schedule: - wait_until_scheduled(start_monotonic, replay_start, event.replay_timestamp) - hub_ws.publish_locations([event.location]) + replay_batches = build_replay_batches(replay_schedule, args.batch_window_ms / 1000.0, args.max_batch_size) + for batch in replay_batches: + wait_until_scheduled(start_monotonic, replay_start, batch[0].replay_timestamp) + hub_ws.publish_locations([event.location for event in batch]) LOGGER.debug( - "published replay event synthetic=%s source=%s timestamp=%s", - event.synthetic, - event.location.get("source"), - event.location.get("timestamp_generated"), + "published replay batch size=%d first_timestamp=%s last_timestamp=%s", + len(batch), + batch[0].location.get("timestamp_generated"), + batch[-1].location.get("timestamp_generated"), ) except KeyboardInterrupt: LOGGER.info("stopping replay connector") @@ -166,6 +183,24 @@ def wait_until_scheduled(start_monotonic: float, replay_start: datetime, replay_ time.sleep(min(remaining, 0.25)) +def build_replay_batches(replay_schedule: list, batch_window_seconds: float, max_batch_size: int) -> list[list]: + if not replay_schedule: + return [] + batches: list[list] = [] + current_batch = [replay_schedule[0]] + batch_start = replay_schedule[0].replay_timestamp + for event in replay_schedule[1:]: + same_window = (event.replay_timestamp - batch_start).total_seconds() <= batch_window_seconds + if same_window and len(current_batch) < max_batch_size: + current_batch.append(event) + continue + batches.append(current_batch) + current_batch = [event] + batch_start = event.replay_timestamp + batches.append(current_batch) + return batches + + def require_env(name: str) -> str: value = os.getenv(name) if not value: diff --git a/docs/architecture.md b/docs/architecture.md index 77d468c..89d2d8c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -27,17 +27,19 @@ ## Event Fan-Out 1. REST, MQTT, or WebSocket ingest enters the shared hub service. -2. The hub validates, normalizes, deduplicates, updates in-memory transient state, and emits the native location event on the ingest path. -3. A buffered in-process derived-location worker handles non-critical follow-on work such as alternate-CRS publication, geofence evaluation, and optional collision evaluation. -4. When that derived queue is full, the hub drops derived work for newer locations rather than backpressuring the ingest path. -5. MQTT and WebSocket consume the resulting internal event stream and publish transport-specific payloads. +2. The hub validates, normalizes, deduplicates, and updates in-memory transient state on the ingest path. +3. A buffered native-publication stage emits native location and motion events without blocking ingest on downstream fan-out. +4. A second buffered decision stage is the insertion point for future filtered or smoothed track processing and currently drives alternate-CRS publication, geofence evaluation, and optional collision evaluation. +5. MQTT and WebSocket consume the resulting internal event stream and publish transport-specific payloads in batches. +6. When any non-critical queue fills, the hub drops newer work on that path rather than backpressuring raw ingest. Implications: - ingest logic is shared across REST, MQTT, and WebSocket - MQTT is no longer the only downstream publication path - the internal event seam decouples downstream publication from MQTT-specific topics -- location ingest latency is protected from slower geofence or collision work -- the derived-location queue is the intended insertion point for future filtered or smoothed track processing before fence/collision decisions +- location ingest latency is protected from slower transport fan-out, geofence work, or collision work +- the decision-stage queue is the intended insertion point for future filtered or smoothed track processing before fence/collision decisions +- WebSocket fan-out coalesces multiple internal events into fewer wrapper messages and drops outbound payloads for slow subscribers instead of tearing the connection down immediately - hub-issued UUIDs for REST-managed resources, derived fence/collision events, and RPC caller IDs now use UUIDv7 so emitted identifiers are time-sortable - internal hub events carry the persisted `origin_hub_id` so downstream transports preserve source provenance diff --git a/docs/configuration.md b/docs/configuration.md index 898c05f..c3708c5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -19,7 +19,9 @@ Runtime lifecycle behavior: - `WEBSOCKET_WRITE_TIMEOUT` (duration, default `5s`) - `WEBSOCKET_READ_TIMEOUT` (duration, default `1m`) - `WEBSOCKET_PING_INTERVAL` (duration, default `30s`) -- `WEBSOCKET_OUTBOUND_BUFFER` (default `32`) +- `WEBSOCKET_OUTBOUND_BUFFER` (default `256`) +- `EVENT_BUS_SUBSCRIBER_BUFFER` (default `1024`) +- `NATIVE_LOCATION_BUFFER` (default `2048`) - `DERIVED_LOCATION_BUFFER` (default `1024`) Hub metadata bootstrap behavior: @@ -52,12 +54,14 @@ Stateful ingest behavior: - latest provider-source location state, trackable latest-location state, proximity hysteresis state, fence membership state, and collision pair state are all kept in process memory with the configured expiry semantics - metadata is loaded from Postgres at startup, updated immediately after successful CRUD writes, and reconciled in the background every `METADATA_RECONCILE_INTERVAL` - durable hub metadata is also loaded from Postgres at startup before the service begins accepting traffic -- WebSocket delivery uses a per-connection outbound queue capped by `WEBSOCKET_OUTBOUND_BUFFER`; slow subscribers are disconnected instead of backpressuring the ingest path +- WebSocket delivery uses a per-connection outbound queue capped by `WEBSOCKET_OUTBOUND_BUFFER`; when that queue fills, outbound payloads are dropped instead of backpressuring ingest or disconnecting the subscriber - WebSocket liveness uses server ping frames every `WEBSOCKET_PING_INTERVAL` and considers the connection stale when no inbound message or pong arrives before `WEBSOCKET_READ_TIMEOUT` -- non-critical derived location work such as alternate-CRS publication, geofence evaluation, and collision evaluation is queued behind `DERIVED_LOCATION_BUFFER` -- when the derived queue is full, the hub drops new derived work instead of slowing raw location ingest +- internal event-bus subscribers such as MQTT and WebSocket consume behind `EVENT_BUS_SUBSCRIBER_BUFFER` +- native location publication is queued behind `NATIVE_LOCATION_BUFFER` so ingest can decouple from transport fan-out +- post-native decision work such as future filtering, alternate-CRS publication, geofence evaluation, and collision evaluation is queued behind `DERIVED_LOCATION_BUFFER` +- when the native, decision, event-bus, or outbound socket queues are full, the hub drops newer work on those non-critical paths instead of slowing raw location ingest - the `metadata_changes` WebSocket topic emits lightweight `{id,type,operation,timestamp}` notifications for zone, fence, trackable, and location-provider CRUD or reconcile drift -- when `COLLISIONS_ENABLED=true`, the hub evaluates trackable-versus-trackable collisions from the latest WGS84 motion state and keeps short-lived collision pair state in memory for `COLLISION_STATE_TTL` +- when `COLLISIONS_ENABLED=true`, the hub evaluates trackable-versus-trackable collisions from the latest active WGS84 motion state and keeps short-lived collision pair state in memory for `COLLISION_STATE_TTL` - `COLLISION_COLLIDING_DEBOUNCE` limits repeated `colliding` emissions for already-active pairs RPC behavior: diff --git a/internal/config/config.go b/internal/config/config.go index 7f161b5..58659d4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,8 @@ type Config struct { WebSocketReadTimeout time.Duration WebSocketPingInterval time.Duration WebSocketOutboundBuffer int + EventBusSubscriberBuffer int + NativeLocationBuffer int DerivedLocationBuffer int StateLocationTTL time.Duration StateProximityTTL time.Duration @@ -82,7 +84,9 @@ func fromLookupEnv(lookup lookupEnvFunc) (Config, error) { WebSocketWriteTimeout: durationEnvWithLookup(lookup, "WEBSOCKET_WRITE_TIMEOUT", 5*time.Second), WebSocketReadTimeout: durationEnvWithLookup(lookup, "WEBSOCKET_READ_TIMEOUT", time.Minute), WebSocketPingInterval: durationEnvWithLookup(lookup, "WEBSOCKET_PING_INTERVAL", 30*time.Second), - WebSocketOutboundBuffer: intEnvWithLookup(lookup, "WEBSOCKET_OUTBOUND_BUFFER", 32), + WebSocketOutboundBuffer: intEnvWithLookup(lookup, "WEBSOCKET_OUTBOUND_BUFFER", 256), + EventBusSubscriberBuffer: intEnvWithLookup(lookup, "EVENT_BUS_SUBSCRIBER_BUFFER", 1024), + NativeLocationBuffer: intEnvWithLookup(lookup, "NATIVE_LOCATION_BUFFER", 2048), DerivedLocationBuffer: intEnvWithLookup(lookup, "DERIVED_LOCATION_BUFFER", 1024), StateLocationTTL: durationEnvWithLookup(lookup, "STATE_LOCATION_TTL", 10*time.Minute), StateProximityTTL: durationEnvWithLookup(lookup, "STATE_PROXIMITY_TTL", 5*time.Minute), @@ -146,6 +150,12 @@ func fromLookupEnv(lookup lookupEnvFunc) (Config, error) { if cfg.WebSocketOutboundBuffer <= 0 { return Config{}, fmt.Errorf("WEBSOCKET_OUTBOUND_BUFFER must be > 0") } + if cfg.EventBusSubscriberBuffer <= 0 { + return Config{}, fmt.Errorf("EVENT_BUS_SUBSCRIBER_BUFFER must be > 0") + } + if cfg.NativeLocationBuffer <= 0 { + return Config{}, fmt.Errorf("NATIVE_LOCATION_BUFFER must be > 0") + } if cfg.DerivedLocationBuffer <= 0 { return Config{}, fmt.Errorf("DERIVED_LOCATION_BUFFER must be > 0") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4e62236..e9cd731 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -42,6 +42,9 @@ func TestDefaults(t *testing.T) { if cfg.WebSocketWriteTimeout <= 0 || cfg.WebSocketReadTimeout <= 0 || cfg.WebSocketPingInterval <= 0 || cfg.WebSocketOutboundBuffer <= 0 { t.Fatal("expected positive websocket settings") } + if cfg.EventBusSubscriberBuffer <= 0 || cfg.NativeLocationBuffer <= 0 { + t.Fatal("expected positive async buffer settings") + } if cfg.DerivedLocationBuffer <= 0 { t.Fatal("expected positive derived location buffer") } @@ -127,6 +130,30 @@ func TestDerivedLocationBufferMustBePositive(t *testing.T) { } } +func TestNativeLocationBufferMustBePositive(t *testing.T) { + t.Parallel() + + _, err := configFromMap(map[string]string{ + "AUTH_MODE": "none", + "NATIVE_LOCATION_BUFFER": "0", + }) + if err == nil { + t.Fatal("expected validation error") + } +} + +func TestEventBusSubscriberBufferMustBePositive(t *testing.T) { + t.Parallel() + + _, err := configFromMap(map[string]string{ + "AUTH_MODE": "none", + "EVENT_BUS_SUBSCRIBER_BUFFER": "0", + }) + if err == nil { + t.Fatal("expected validation error") + } +} + func TestValidHubIDLoadsSuccessfully(t *testing.T) { t.Parallel() diff --git a/internal/hub/derived_processor.go b/internal/hub/derived_processor.go index a61955a..83b5a1f 100644 --- a/internal/hub/derived_processor.go +++ b/internal/hub/derived_processor.go @@ -18,20 +18,33 @@ type derivedLocationSubmitter interface { } type derivedLocationProcessor struct { - service *Service - logger *zap.Logger - queue chan derivedLocationWork - dropped atomic.Uint64 + service *Service + logger *zap.Logger + queue chan derivedLocationWork + label string + onDrop func() uint64 + onProcess func(context.Context, gen.Location) error + dropped atomic.Uint64 } -func startDerivedLocationProcessor(ctx context.Context, service *Service, buffer int) derivedLocationSubmitter { +func startDerivedLocationProcessor( + ctx context.Context, + service *Service, + buffer int, + label string, + onDrop func() uint64, + onProcess func(context.Context, gen.Location) error, +) derivedLocationSubmitter { if service == nil || buffer <= 0 { return nil } processor := &derivedLocationProcessor{ - service: service, - logger: service.logger, - queue: make(chan derivedLocationWork, buffer), + service: service, + logger: service.logger, + queue: make(chan derivedLocationWork, buffer), + label: label, + onDrop: onDrop, + onProcess: onProcess, } go processor.run(ctx) return processor @@ -42,8 +55,11 @@ func (p *derivedLocationProcessor) Submit(work derivedLocationWork) { case p.queue <- work: default: dropped := p.dropped.Add(1) + if p.onDrop != nil { + dropped = p.onDrop() + } if dropped == 1 || dropped%100 == 0 { - p.logger.Warn("derived location queue full; dropping derived work", zap.Uint64("dropped", dropped)) + p.logger.Warn(p.label+" full; dropping location work", zap.Uint64("dropped", dropped)) } } } @@ -54,13 +70,23 @@ func (p *derivedLocationProcessor) run(ctx context.Context) { case <-ctx.Done(): return case work := <-p.queue: - if err := p.service.processDerivedLocation(ctx, work.Location); err != nil { - p.logger.Warn("derived location processing failed", zap.Error(err), zap.String("provider_id", work.Location.ProviderId), zap.String("source", work.Location.Source)) + if err := p.onProcess(ctx, work.Location); err != nil { + p.logger.Warn(p.label+" processing failed", zap.Error(err), zap.String("provider_id", work.Location.ProviderId), zap.String("source", work.Location.Source)) } } } } +type decisionLocationStage interface { + Process(context.Context, gen.Location) (gen.Location, bool, error) +} + +type passthroughDecisionStage struct{} + +func (passthroughDecisionStage) Process(_ context.Context, location gen.Location) (gen.Location, bool, error) { + return location, true, nil +} + type derivedLocationView struct { service *Service location gen.Location diff --git a/internal/hub/events.go b/internal/hub/events.go index e902cd5..a0ec13f 100644 --- a/internal/hub/events.go +++ b/internal/hub/events.go @@ -119,11 +119,12 @@ type EventBus struct { mu sync.RWMutex nextID int subscribers map[int]chan Event + stats *RuntimeStats } // NewEventBus constructs an EventBus. func NewEventBus() *EventBus { - return &EventBus{subscribers: map[int]chan Event{}} + return &EventBus{subscribers: map[int]chan Event{}, stats: newRuntimeStats()} } // Subscribe registers a buffered event subscriber. @@ -148,6 +149,14 @@ func (b *EventBus) Subscribe(buffer int) (<-chan Event, func()) { } } +// Stats returns the counters associated with this bus. +func (b *EventBus) Stats() *RuntimeStats { + if b == nil { + return nil + } + return b.stats +} + // Emit publishes an event to all subscribers. func (b *EventBus) Emit(event Event) { b.mu.RLock() @@ -161,6 +170,9 @@ func (b *EventBus) Emit(event Event) { select { case ch <- event: default: + if b.stats != nil { + b.stats.IncEventBusDrops() + } } } } diff --git a/internal/hub/processing_state.go b/internal/hub/processing_state.go index ae15f48..1355ee3 100644 --- a/internal/hub/processing_state.go +++ b/internal/hub/processing_state.go @@ -177,6 +177,21 @@ func (s *ProcessingState) DeleteMotion(trackableID string) { delete(s.motions, trackableID) } +func (s *ProcessingState) ListActiveMotions() []gen.TrackableMotion { + s.mu.Lock() + defer s.mu.Unlock() + now := s.nowUTC() + motions := make([]gen.TrackableMotion, 0, len(s.motions)) + for key, item := range s.motions { + if !item.expiresAt.After(now) { + delete(s.motions, key) + continue + } + motions = append(motions, item.value) + } + return motions +} + func (s *ProcessingState) GetCollisionState(key string) (activeCollisionState, bool) { s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/hub/runtime_stats.go b/internal/hub/runtime_stats.go new file mode 100644 index 0000000..98c43e9 --- /dev/null +++ b/internal/hub/runtime_stats.go @@ -0,0 +1,64 @@ +package hub + +import "sync/atomic" + +// RuntimeStats tracks hot-path overload conditions so callers can distinguish +// ingest bottlenecks from downstream delivery loss. +type RuntimeStats struct { + eventBusDrops atomic.Uint64 + nativeQueueDrops atomic.Uint64 + decisionQueueDrops atomic.Uint64 + websocketOutboundDrops atomic.Uint64 +} + +// RuntimeStatsSnapshot is a stable copy of runtime counters. +type RuntimeStatsSnapshot struct { + EventBusDrops uint64 `json:"event_bus_drops"` + NativeQueueDrops uint64 `json:"native_queue_drops"` + DecisionQueueDrops uint64 `json:"decision_queue_drops"` + WebSocketOutboundDrops uint64 `json:"websocket_outbound_drops"` +} + +func newRuntimeStats() *RuntimeStats { + return &RuntimeStats{} +} + +func (s *RuntimeStats) Snapshot() RuntimeStatsSnapshot { + if s == nil { + return RuntimeStatsSnapshot{} + } + return RuntimeStatsSnapshot{ + EventBusDrops: s.eventBusDrops.Load(), + NativeQueueDrops: s.nativeQueueDrops.Load(), + DecisionQueueDrops: s.decisionQueueDrops.Load(), + WebSocketOutboundDrops: s.websocketOutboundDrops.Load(), + } +} + +func (s *RuntimeStats) IncEventBusDrops() uint64 { + if s == nil { + return 0 + } + return s.eventBusDrops.Add(1) +} + +func (s *RuntimeStats) IncNativeQueueDrops() uint64 { + if s == nil { + return 0 + } + return s.nativeQueueDrops.Add(1) +} + +func (s *RuntimeStats) IncDecisionQueueDrops() uint64 { + if s == nil { + return 0 + } + return s.decisionQueueDrops.Add(1) +} + +func (s *RuntimeStats) IncWebSocketOutboundDrops() uint64 { + if s == nil { + return 0 + } + return s.websocketOutboundDrops.Add(1) +} diff --git a/internal/hub/service.go b/internal/hub/service.go index fad34b9..0c74032 100644 --- a/internal/hub/service.go +++ b/internal/hub/service.go @@ -31,6 +31,7 @@ type Config struct { LocationTTL time.Duration ProximityTTL time.Duration DedupTTL time.Duration + NativeLocationBuffer int DerivedLocationBuffer int ProximityResolutionEntryConfidenceMin float64 ProximityResolutionExitGraceDuration time.Duration @@ -57,7 +58,10 @@ type Service struct { transformCache *transform.Cache metadata *MetadataCache state *ProcessingState + stats *RuntimeStats + nativeQueue derivedLocationSubmitter derivedQueue derivedLocationSubmitter + decisionStage decisionLocationStage } // HTTPError represents an API error that should be rendered with a specific @@ -79,8 +83,11 @@ func (s *Service) Start(ctx context.Context) { return } s.processingState().StartSweeper(ctx, time.Second) + if s.nativeQueue == nil { + s.nativeQueue = startDerivedLocationProcessor(ctx, s, s.cfg.NativeLocationBuffer, "native location queue", s.stats.IncNativeQueueDrops, s.processNativeLocation) + } if s.derivedQueue == nil { - s.derivedQueue = startDerivedLocationProcessor(ctx, s, s.cfg.DerivedLocationBuffer) + s.derivedQueue = startDerivedLocationProcessor(ctx, s, s.cfg.DerivedLocationBuffer, "decision location queue", s.stats.IncDecisionQueueDrops, s.processDecisionLocation) } if s.metadata == nil || s.queries == nil { return @@ -110,6 +117,9 @@ func New(logger *zap.Logger, queries sqlcgen.Querier, bus *EventBus, cfg Config) if cfg.MetadataReconcileInterval <= 0 { cfg.MetadataReconcileInterval = 30 * time.Second } + if cfg.NativeLocationBuffer <= 0 { + cfg.NativeLocationBuffer = 2048 + } if cfg.DerivedLocationBuffer <= 0 { cfg.DerivedLocationBuffer = 1024 } @@ -128,9 +138,18 @@ func New(logger *zap.Logger, queries sqlcgen.Querier, bus *EventBus, cfg Config) transformCache: transform.NewCache(), metadata: metadata, state: NewProcessingState(nowFn), + stats: runtimeStatsFromBus(bus), + decisionStage: passthroughDecisionStage{}, }, nil } +func runtimeStatsFromBus(bus *EventBus) *RuntimeStats { + if bus != nil && bus.Stats() != nil { + return bus.Stats() + } + return newRuntimeStats() +} + type proximityResolutionPolicy struct { EntryConfidenceMin float64 ExitGraceDuration time.Duration @@ -899,14 +918,8 @@ func (s *Service) recordLocation(ctx context.Context, location gen.Location, ttl s.processingState().SetTrackableLocation(latestTrackableLocationKey(trackableID), location, ttl) } } - if s.derivedQueue != nil { - if err := s.publishNativeLocation(ctx, location); err != nil { - s.logger.Warn("location event emit failed", zap.Error(err), zap.String("provider_id", location.ProviderId)) - } - if _, err := s.publishNativeTrackableMotions(ctx, location); err != nil { - s.logger.Warn("trackable motion event emit failed", zap.Error(err), zap.String("provider_id", location.ProviderId)) - } - s.derivedQueue.Submit(derivedLocationWork{Location: location}) + if s.nativeQueue != nil { + s.nativeQueue.Submit(derivedLocationWork{Location: location}) return nil } if err := s.publishLocation(ctx, location); err != nil { @@ -927,6 +940,27 @@ func (s *Service) recordLocation(ctx context.Context, location gen.Location, ttl return nil } +func (s *Service) processNativeLocation(ctx context.Context, location gen.Location) error { + if err := s.publishNativeLocation(ctx, location); err != nil { + return err + } + if _, err := s.publishNativeTrackableMotions(ctx, location); err != nil { + return err + } + if s.derivedQueue != nil { + s.derivedQueue.Submit(derivedLocationWork{Location: location}) + } + return nil +} + +func (s *Service) processDecisionLocation(ctx context.Context, location gen.Location) error { + decisionLocation, ok, err := s.decisionStage.Process(ctx, location) + if err != nil || !ok { + return err + } + return s.processDerivedLocation(ctx, decisionLocation) +} + func (s *Service) processDerivedLocation(ctx context.Context, location gen.Location) error { if s.bus == nil { return nil @@ -1365,26 +1399,30 @@ func (s *Service) publishCollisionEvents(ctx context.Context, motions []gen.Trac if s.bus == nil || !s.cfg.CollisionsEnabled || len(motions) == 0 { return nil } - allTrackables, err := s.ListTrackables(ctx) - if err != nil { - return err - } - trackables := make(map[string]gen.Trackable, len(allTrackables)) - for _, trackable := range allTrackables { - trackables[trackable.Id.String()] = trackable + activeMotions := s.processingState().ListActiveMotions() + activeMotionByID := make(map[string]gen.TrackableMotion, len(activeMotions)) + for _, motion := range activeMotions { + activeMotionByID[motion.Id] = motion } for _, motion := range motions { s.processingState().SetMotion(motion.Id, motion, s.cfg.CollisionStateTTL) - for otherID, otherTrackable := range trackables { + leftTrackable, err := s.trackableByID(ctx, motion.Id) + if err != nil { + continue + } + for otherID, otherMotion := range activeMotionByID { if otherID == motion.Id { continue } - otherMotion, ok := s.processingState().GetMotion(otherID) - if !ok { + otherTrackable, err := s.trackableByID(ctx, otherID) + if err != nil { + continue + } + if !motionsMayCollide(motion, leftTrackable, otherMotion, otherTrackable) { s.processingState().DeleteCollisionState(collisionPairKey(motion.Id, otherID)) continue } - event, active, err := s.evaluateCollision(motion, trackables[motion.Id], otherMotion, otherTrackable) + event, active, err := s.evaluateCollision(motion, leftTrackable, otherMotion, otherTrackable) if err != nil { return err } @@ -1418,6 +1456,19 @@ func (s *Service) publishCollisionEvents(ctx context.Context, motions []gen.Trac return nil } +func (s *Service) trackableByID(ctx context.Context, id string) (gen.Trackable, error) { + if cache := s.metadataCache(); cache != nil { + if trackable, ok := cache.TrackableByID(id); ok { + return trackable, nil + } + } + parsed, err := uuid.Parse(id) + if err != nil { + return gen.Trackable{}, err + } + return s.GetTrackable(ctx, openapi_types.UUID(parsed)) +} + type activeCollisionState struct { Active bool `json:"active"` StartTime time.Time `json:"start_time"` @@ -1515,6 +1566,19 @@ func motionsCollide(leftMotion gen.TrackableMotion, leftTrackable gen.Trackable, return false, nil, distance } +func motionsMayCollide(leftMotion gen.TrackableMotion, leftTrackable gen.Trackable, rightMotion gen.TrackableMotion, rightTrackable gen.Trackable) bool { + leftPoint, err := point2D(leftMotion.Location.Position) + if err != nil { + return false + } + rightPoint, err := point2D(rightMotion.Location.Position) + if err != nil { + return false + } + maxDistance := effectiveRadius(leftTrackable) + effectiveRadius(rightTrackable) + return math.Abs(leftPoint[0]-rightPoint[0]) <= maxDistance && math.Abs(leftPoint[1]-rightPoint[1]) <= maxDistance +} + func collisionPairKey(leftID, rightID string) string { ids := []string{leftID, rightID} sort.Strings(ids) diff --git a/internal/hub/service_test.go b/internal/hub/service_test.go index 54b8a82..4c73abe 100644 --- a/internal/hub/service_test.go +++ b/internal/hub/service_test.go @@ -600,7 +600,41 @@ func (c *capturingDerivedSubmitter) Submit(work derivedLocationWork) { c.works = append(c.works, work) } -func TestRecordLocationWithDerivedQueuePublishesNativeAndQueuesDerivedWork(t *testing.T) { +func TestRecordLocationWithNativeQueueQueuesWork(t *testing.T) { + t.Parallel() + + bus := NewEventBus() + queue := &capturingDerivedSubmitter{} + crs := "EPSG:4326" + location := testLocationWithCoordinates(t, &crs, "external-source", [2]float32{8.5, 47.3}) + trackables := []string{"trackable-a"} + location.Trackables = &trackables + + service := &Service{ + bus: bus, + nativeQueue: queue, + cfg: Config{ + NativeLocationBuffer: 16, + LocationTTL: time.Minute, + DedupTTL: time.Minute, + CollisionStateTTL: time.Minute, + CollisionCollidingDebounce: time.Second, + MetadataReconcileInterval: time.Second, + }, + metadata: &MetadataCache{snapshot: newMetadataSnapshot(nil, nil, nil, nil)}, + state: NewProcessingState(time.Now), + logger: zapTestLogger(t), + } + + if err := service.recordLocation(context.Background(), location, time.Minute); err != nil { + t.Fatalf("recordLocation failed: %v", err) + } + if len(queue.works) != 1 { + t.Fatalf("expected one queued native work item, got %d", len(queue.works)) + } +} + +func TestProcessNativeLocationPublishesNativeAndQueuesDecisionWork(t *testing.T) { t.Parallel() bus := NewEventBus() @@ -616,20 +650,20 @@ func TestRecordLocationWithDerivedQueuePublishesNativeAndQueuesDerivedWork(t *te bus: bus, derivedQueue: queue, cfg: Config{ + DerivedLocationBuffer: 16, LocationTTL: time.Minute, DedupTTL: time.Minute, CollisionStateTTL: time.Minute, CollisionCollidingDebounce: time.Second, MetadataReconcileInterval: time.Second, - DerivedLocationBuffer: 16, }, metadata: &MetadataCache{snapshot: newMetadataSnapshot(nil, nil, nil, nil)}, state: NewProcessingState(time.Now), logger: zapTestLogger(t), } - if err := service.recordLocation(context.Background(), location, time.Minute); err != nil { - t.Fatalf("recordLocation failed: %v", err) + if err := service.processNativeLocation(context.Background(), location); err != nil { + t.Fatalf("processNativeLocation failed: %v", err) } events := collectEvents(ch, 2) if len(events) != 2 { @@ -639,7 +673,7 @@ func TestRecordLocationWithDerivedQueuePublishesNativeAndQueuesDerivedWork(t *te t.Fatalf("unexpected native location source: %s", got.Source) } if len(queue.works) != 1 { - t.Fatalf("expected one queued derived work item, got %d", len(queue.works)) + t.Fatalf("expected one queued decision work item, got %d", len(queue.works)) } } diff --git a/internal/ws/hub.go b/internal/ws/hub.go index 1a2df12..7dbb0be 100644 --- a/internal/ws/hub.go +++ b/internal/ws/hub.go @@ -17,20 +17,22 @@ import ( ) const ( - errUnknownEvent = 10000 - errUnknownTopic = 10001 - errSubscribeFailed = 10002 - errUnsubscribeFailed = 10003 - errUnauthorized = 10004 - errInvalidPayload = 10005 - topicLocationUpdates = "location_updates" - topicLocationGeoJSON = "location_updates:geojson" - topicCollisionEvents = "collision_events" - topicFenceEvents = "fence_events" - topicFenceGeoJSON = "fence_events:geojson" - topicTrackableMotions = "trackable_motions" - topicProximityUpdates = "proximity_updates" - topicMetadataChanges = "metadata_changes" + errUnknownEvent = 10000 + errUnknownTopic = 10001 + errSubscribeFailed = 10002 + errUnsubscribeFailed = 10003 + errUnauthorized = 10004 + errInvalidPayload = 10005 + topicLocationUpdates = "location_updates" + topicLocationGeoJSON = "location_updates:geojson" + topicCollisionEvents = "collision_events" + topicFenceEvents = "fence_events" + topicFenceGeoJSON = "fence_events:geojson" + topicTrackableMotions = "trackable_motions" + topicProximityUpdates = "proximity_updates" + topicMetadataChanges = "metadata_changes" + eventDispatchBatchSize = 256 + eventDispatchFlushWait = 5 * time.Millisecond ) type wrapper struct { @@ -56,6 +58,7 @@ type Hub struct { readTimeout time.Duration pingInterval time.Duration upgrader websocket.Upgrader + stats *hub.RuntimeStats mu sync.RWMutex connections map[*connection]struct{} } @@ -122,7 +125,7 @@ type metadataFilter struct { } // New constructs a WebSocket hub and starts bus fan-out. -func New(logger *zap.Logger, service *hub.Service, bus *hub.EventBus, authenticator auth.Authenticator, registry *auth.Registry, authCfg config.AuthConfig, writeTimeout, readTimeout, pingInterval time.Duration, outboundBuffer int, collisionsEnabled bool) *Hub { +func New(logger *zap.Logger, service *hub.Service, bus *hub.EventBus, authenticator auth.Authenticator, registry *auth.Registry, authCfg config.AuthConfig, writeTimeout, readTimeout, pingInterval time.Duration, outboundBuffer, subscriberBuffer int, collisionsEnabled bool) *Hub { if writeTimeout <= 0 { writeTimeout = 5 * time.Second } @@ -147,18 +150,26 @@ func New(logger *zap.Logger, service *hub.Service, bus *hub.EventBus, authentica collisionsEnabled: collisionsEnabled, readTimeout: readTimeout, pingInterval: pingInterval, + stats: runtimeStatsFromBus(bus), upgrader: websocket.Upgrader{ CheckOrigin: func(*http.Request) bool { return true }, }, connections: map[*connection]struct{}{}, } if bus != nil { - ch, _ := bus.Subscribe(128) + ch, _ := bus.Subscribe(subscriberBuffer) go h.consume(ch) } return h } +func runtimeStatsFromBus(bus *hub.EventBus) *hub.RuntimeStats { + if bus != nil && bus.Stats() != nil { + return bus.Stats() + } + return nil +} + // Handle upgrades the request to a WebSocket connection and serves the OMLOX // wrapper protocol on it. func (h *Hub) Handle(w http.ResponseWriter, r *http.Request) { @@ -184,12 +195,53 @@ func (h *Hub) Handle(w http.ResponseWriter, r *http.Request) { } func (h *Hub) consume(ch <-chan hub.Event) { - for event := range ch { - h.broadcast(event) + batch := make([]hub.Event, 0, eventDispatchBatchSize) + timer := time.NewTimer(time.Hour) + timer.Stop() + timerActive := false + flush := func() { + if len(batch) == 0 { + return + } + out := append([]hub.Event(nil), batch...) + batch = batch[:0] + h.broadcastBatch(out) + } + for { + select { + case event, ok := <-ch: + if !ok { + flush() + return + } + batch = append(batch, event) + if len(batch) == 1 && !timerActive { + timer.Reset(eventDispatchFlushWait) + timerActive = true + } + if len(batch) >= eventDispatchBatchSize { + if timerActive { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timerActive = false + } + flush() + } + case <-timer.C: + timerActive = false + flush() + } } } -func (h *Hub) broadcast(event hub.Event) { +func (h *Hub) broadcastBatch(events []hub.Event) { + if len(events) == 0 { + return + } h.mu.RLock() conns := make([]*connection, 0, len(h.connections)) for conn := range h.connections { @@ -197,7 +249,7 @@ func (h *Hub) broadcast(event hub.Event) { } h.mu.RUnlock() for _, conn := range conns { - conn.deliver(event) + conn.deliverBatch(events) } } @@ -384,7 +436,7 @@ func (c *connection) authenticate(params map[string]any, subscribe bool, topic s return principal, nil } -func (c *connection) deliver(event hub.Event) { +func (c *connection) deliverBatch(events []hub.Event) { c.mu.RLock() subs := make([]subscription, 0, len(c.subs)) for _, sub := range c.subs { @@ -392,7 +444,7 @@ func (c *connection) deliver(event hub.Event) { } c.mu.RUnlock() for _, sub := range subs { - payload, ok := payloadForSubscription(sub, event) + payload, ok := payloadBatchForSubscription(sub, events) if !ok { continue } @@ -416,8 +468,10 @@ func (c *connection) sendWrapper(msg wrapper) { return case c.send <- raw: default: - c.hub.logger.Debug("websocket outbound buffer full; closing slow connection") - c.close() + if c.hub.stats != nil { + c.hub.stats.IncWebSocketOutboundDrops() + } + c.hub.logger.Debug("websocket outbound buffer full; dropping outbound payload") } } @@ -453,82 +507,139 @@ func parseFilter(topic string, params map[string]any) (any, error) { } } -func payloadForSubscription(sub subscription, event hub.Event) (json.RawMessage, bool) { +func payloadBatchForSubscription(sub subscription, events []hub.Event) (json.RawMessage, bool) { switch sub.topic { case topicLocationUpdates: - if event.Kind != hub.EventLocation { - return nil, false + items := make([]gen.Location, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventLocation { + continue + } + envelope, err := hub.Decode[hub.LocationEnvelope](event) + if err != nil || !matchLocation(sub.filter.(locationFilter), envelope.Location) { + continue + } + items = append(items, envelope.Location) } - envelope, err := hub.Decode[hub.LocationEnvelope](event) - if err != nil || !matchLocation(sub.filter.(locationFilter), envelope.Location) { + if len(items) == 0 { return nil, false } - return marshalPayload([]gen.Location{envelope.Location}) + return marshalPayload(items) case topicLocationGeoJSON: - if event.Kind != hub.EventLocation { - return nil, false + items := make([]hub.GeoJSONFeatureCollection, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventLocation { + continue + } + envelope, err := hub.Decode[hub.LocationEnvelope](event) + if err != nil || !matchLocation(sub.filter.(locationFilter), envelope.Location) { + continue + } + items = append(items, envelope.GeoJSON) } - envelope, err := hub.Decode[hub.LocationEnvelope](event) - if err != nil || !matchLocation(sub.filter.(locationFilter), envelope.Location) { + if len(items) == 0 { return nil, false } - return marshalPayload([]hub.GeoJSONFeatureCollection{envelope.GeoJSON}) + return marshalPayload(items) case topicProximityUpdates: - if event.Kind != hub.EventProximity { - return nil, false + items := make([]gen.Proximity, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventProximity { + continue + } + envelope, err := hub.Decode[hub.ProximityEnvelope](event) + if err != nil { + continue + } + items = append(items, envelope.Proximity) } - envelope, err := hub.Decode[hub.ProximityEnvelope](event) - if err != nil { + if len(items) == 0 { return nil, false } - return marshalPayload([]gen.Proximity{envelope.Proximity}) + return marshalPayload(items) case topicTrackableMotions: - if event.Kind != hub.EventTrackableMotion { - return nil, false + items := make([]gen.TrackableMotion, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventTrackableMotion { + continue + } + envelope, err := hub.Decode[hub.TrackableMotionEnvelope](event) + if err != nil || !matchMotion(sub.filter.(motionFilter), envelope.Motion) { + continue + } + items = append(items, envelope.Motion) } - envelope, err := hub.Decode[hub.TrackableMotionEnvelope](event) - if err != nil || !matchMotion(sub.filter.(motionFilter), envelope.Motion) { + if len(items) == 0 { return nil, false } - return marshalPayload([]gen.TrackableMotion{envelope.Motion}) + return marshalPayload(items) case topicFenceEvents: - if event.Kind != hub.EventFenceEvent { - return nil, false + items := make([]gen.FenceEvent, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventFenceEvent { + continue + } + envelope, err := hub.Decode[hub.FenceEventEnvelope](event) + if err != nil || !matchFence(sub.filter.(fenceFilter), envelope.Event) { + continue + } + items = append(items, envelope.Event) } - envelope, err := hub.Decode[hub.FenceEventEnvelope](event) - if err != nil || !matchFence(sub.filter.(fenceFilter), envelope.Event) { + if len(items) == 0 { return nil, false } - return marshalPayload([]gen.FenceEvent{envelope.Event}) + return marshalPayload(items) case topicFenceGeoJSON: - if event.Kind != hub.EventFenceEvent { - return nil, false + items := make([]hub.GeoJSONFeatureCollection, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventFenceEvent { + continue + } + envelope, err := hub.Decode[hub.FenceEventEnvelope](event) + if err != nil || !matchFence(sub.filter.(fenceFilter), envelope.Event) { + continue + } + items = append(items, envelope.GeoJSON) } - envelope, err := hub.Decode[hub.FenceEventEnvelope](event) - if err != nil || !matchFence(sub.filter.(fenceFilter), envelope.Event) { + if len(items) == 0 { return nil, false } - return marshalPayload([]hub.GeoJSONFeatureCollection{envelope.GeoJSON}) + return marshalPayload(items) case topicCollisionEvents: - if event.Kind != hub.EventCollisionEvent { - return nil, false + items := make([]gen.CollisionEvent, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventCollisionEvent { + continue + } + envelope, err := hub.Decode[hub.CollisionEnvelope](event) + if err != nil || !matchCollision(sub.filter.(collisionFilter), envelope.Event) { + continue + } + items = append(items, envelope.Event) } - envelope, err := hub.Decode[hub.CollisionEnvelope](event) - if err != nil || !matchCollision(sub.filter.(collisionFilter), envelope.Event) { + if len(items) == 0 { return nil, false } - return marshalPayload([]gen.CollisionEvent{envelope.Event}) + return marshalPayload(items) case topicMetadataChanges: - if event.Kind != hub.EventMetadataChange { - return nil, false + items := make([]hub.MetadataChange, 0, len(events)) + for _, event := range events { + if event.Kind != hub.EventMetadataChange { + continue + } + change, err := hub.Decode[hub.MetadataChange](event) + if err != nil || !matchMetadata(sub.filter.(metadataFilter), change) { + continue + } + items = append(items, change) } - change, err := hub.Decode[hub.MetadataChange](event) - if err != nil || !matchMetadata(sub.filter.(metadataFilter), change) { + if len(items) == 0 { return nil, false } - return marshalPayload([]hub.MetadataChange{change}) + return marshalPayload(items) + default: + return nil, false } - return nil, false } func marshalPayload(value any) (json.RawMessage, bool) { diff --git a/internal/ws/hub_test.go b/internal/ws/hub_test.go index 527f5de..6e38309 100644 --- a/internal/ws/hub_test.go +++ b/internal/ws/hub_test.go @@ -146,7 +146,7 @@ func TestSendWrapperSafeDuringConcurrentClose(t *testing.T) { t.Parallel() bus := hub.NewEventBus() - h := New(zap.NewNop(), nil, bus, nil, nil, config.AuthConfig{Enabled: false, Mode: "none"}, time.Second, 3*time.Second, time.Second, 1, true) + h := New(zap.NewNop(), nil, bus, nil, nil, config.AuthConfig{Enabled: false, Mode: "none"}, time.Second, 3*time.Second, time.Second, 1, 32, true) server2 := httptest.NewServer(http.HandlerFunc(h.Handle)) defer server2.Close() wsURL2 := "ws" + strings.TrimPrefix(server2.URL, "http") + "/v2/ws/socket" @@ -196,7 +196,7 @@ func startTestHub(t *testing.T, bus *hub.EventBus, collisionsEnabled bool) (*web func httpHandler(t *testing.T, bus *hub.EventBus, collisionsEnabled bool) http.Handler { t.Helper() - return http.HandlerFunc(New(zap.NewNop(), nil, bus, nil, nil, config.AuthConfig{Enabled: false, Mode: "none"}, time.Second, 3*time.Second, time.Second, 8, collisionsEnabled).Handle) + return http.HandlerFunc(New(zap.NewNop(), nil, bus, nil, nil, config.AuthConfig{Enabled: false, Mode: "none"}, time.Second, 3*time.Second, time.Second, 8, 128, collisionsEnabled).Handle) } func TestReadDeadlineExtendsOnIncomingMessages(t *testing.T) { @@ -214,6 +214,7 @@ func TestReadDeadlineExtendsOnIncomingMessages(t *testing.T) { 80*time.Millisecond, time.Hour, 8, + 128, true, ) server := httptest.NewServer(http.HandlerFunc(h.Handle))