From 916158cbf447b7ee948776ea8b745229ab1016ad Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:53:52 +0200 Subject: [PATCH 01/25] #361 Remove ITileService contract --- .../contracts/tile_service_interface.py | 63 ------------------- 1 file changed, 63 deletions(-) delete mode 100644 src/application/contracts/tile_service_interface.py diff --git a/src/application/contracts/tile_service_interface.py b/src/application/contracts/tile_service_interface.py deleted file mode 100644 index 26678150..00000000 --- a/src/application/contracts/tile_service_interface.py +++ /dev/null @@ -1,63 +0,0 @@ -from abc import ABC, abstractmethod - - -class ITileService(ABC): - @abstractmethod - def lat_lon_to_tile( - self, - lat: float, - lon: float, - zoom: int, - bounding_box: tuple[float, float, float, float] - ) -> tuple[int, int, int]: - """ - Converts latitude and longitude to tile coordinates (z, x, y) at a given zoom level. The input - coordinates are clamped to the given bounding box and to the Web Mercator latitude limits - (±85.05112878). - :param lat: Latitude in degrees. - :param lon: Longitude in degrees. - :param zoom: Zoom level (0-22). - :param bounding_box: Bounding box defined as a tuple (min_lat, min_lon, max_lat, max_lon) used - to clamp the input coordinates. - :return: Tile coordinates (z, x, y) corresponding to the given latitude and longitude at the - specified zoom level. - :rtype: tuple[int, int, int] - """ - raise NotImplementedError - - @abstractmethod - def build_candidate_tiles( - self, - min_lat: float, - min_lon: float, - max_lat: float, - max_lon: float, - zoom: int - ) -> list[tuple[int, int, int]]: - """ - Builds a list of candidate tile coordinates (z, x, y) that cover the bounding box defined by - the given latitude and longitude. - :param min_lat: Minimum latitude in degrees. - :param min_lon: Minimum longitude in degrees. - :param max_lat: Maximum latitude in degrees. - :param max_lon: Maximum longitude in degrees. - :param zoom: Zoom level (0-22). - :return: List of tile coordinates (z, x, y) that cover the bounding box. - :rtype: list[tuple[int, int, int]] - """ - raise NotImplementedError - - @abstractmethod - def load_tiles(self, number_of_tiles: int) -> list[tuple[int, int, int]]: - """ - Loads valid VMT tile coordinates (z, x, y) from a predefined JSON source on disk. The - implementation parses and validates the file, then cycles the loaded tiles so the returned - list has exactly `number_of_tiles` entries. - :param number_of_tiles: Number of tile coordinates to return. Tiles are repeated cyclically - when the source contains fewer than `number_of_tiles` entries. - :return: List of valid tile coordinates (z, x, y) of length `number_of_tiles`. - :rtype: list[tuple[int, int, int]] - :raises ValueError: If the source file is empty, not valid JSON, not a list, or contains - malformed tile entries. - """ - raise NotImplementedError From 39bee54b91683291345fc358d957b473013e8e27 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:53:58 +0200 Subject: [PATCH 02/25] #361 Remove ITileApiService contract --- .../contracts/tile_api_service_interface.py | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 src/application/contracts/tile_api_service_interface.py diff --git a/src/application/contracts/tile_api_service_interface.py b/src/application/contracts/tile_api_service_interface.py deleted file mode 100644 index 9bc70e1a..00000000 --- a/src/application/contracts/tile_api_service_interface.py +++ /dev/null @@ -1,45 +0,0 @@ -from abc import abstractmethod, ABC - -from pmtiles.reader import Reader - - -class ITileApiService(ABC): - @abstractmethod - def fetch_vmt_tile(self, z: int, x: int, y: int) -> bytes | None: - """ - Fetches a tile from the VMT server based on the provided z, x, and y coordinates. Cache headers - are set to bypass any intermediate cache. - :param z: Zoom level of the tile - :param x: X coordinate of the tile - :param y: Y coordinate of the tile - :return: Raw tile bytes, or None when the server returns 404 or an empty response body. - :rtype: bytes | None - :raises RuntimeError: If the VMT server cannot be reached or returns a non-404 error status. - """ - raise NotImplementedError - - @abstractmethod - def fetch_pmtiles_tile(self, reader: Reader, z: int, x: int, y: int) -> bytes | None: - """ - Fetches a tile from a PMTiles archive via the given reader. The reader resolves the tile's byte - range and returns its contents. - :param reader: PMTiles Reader bound to the archive to read from. - :param z: Zoom level of the tile - :param x: X coordinate of the tile - :param y: Y coordinate of the tile - :return: Raw tile bytes, or None when the tile is not present in the archive. - :rtype: bytes | None - """ - raise NotImplementedError - - @abstractmethod - def create_pmtiles_reader(self, pmtiles_url: str) -> Reader: - """ - Creates a PMTiles Reader that reads the archive at the given URL using HTTP range requests. The - underlying byte-source validates that the server returns HTTP 206 Partial Content with the exact - requested byte range. - :param pmtiles_url: HTTP(S) URL of the PMTiles archive. - :return: PMTiles Reader bound to the remote archive. - :rtype: Reader - """ - raise NotImplementedError From 1efc4159cca5f1dcf69010a038c79b068fbc371d Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:00 +0200 Subject: [PATCH 03/25] #361 Remove IMVTService contract --- .../contracts/mvt_service_interface.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 src/application/contracts/mvt_service_interface.py diff --git a/src/application/contracts/mvt_service_interface.py b/src/application/contracts/mvt_service_interface.py deleted file mode 100644 index 297fb65d..00000000 --- a/src/application/contracts/mvt_service_interface.py +++ /dev/null @@ -1,17 +0,0 @@ -from abc import ABC, abstractmethod - - -class IMVTService(ABC): - @abstractmethod - async def get_mvt_tiles(self, z: int, x: int, y: int) -> bytes | None: - """ - Fetches an MVT tile for the buildings layer at the given zoom level (z) and tile coordinates - (x, y). The tile is generated server-side from PostGIS using `ST_AsMVT` and returned as the - raw protobuf payload. - :param z: Zoom level of the tile. - :param x: X coordinate of the tile. - :param y: Y coordinate of the tile. - :return: Raw MVT tile bytes, or None when no features intersect the tile envelope. - :rtype: bytes | None - """ - raise NotImplementedError From e9a217b8c63427ba0508a86c810341fd68484d25 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:01 +0200 Subject: [PATCH 04/25] #361 Remove tile re-exports from contracts __init__ --- src/application/contracts/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/application/contracts/__init__.py b/src/application/contracts/__init__.py index 91c0f8f2..7d620653 100644 --- a/src/application/contracts/__init__.py +++ b/src/application/contracts/__init__.py @@ -12,13 +12,10 @@ from .file_path_service_interface import IFilePathService from .fkb_service_interface import IFKBService from .monitoring_storage_service import IMonitoringStorageService -from .mvt_service_interface import IMVTService from .open_street_map_file_service_interface import IOpenStreetMapFileService from .open_street_map_service_interface import IOpenStreetMapService from .release_service_interface import IReleaseService from .stac_io_service_interface import IStacIOService from .stac_service_interface import IStacService from .test_dataset_service_interface import ITestDatasetService -from .tile_api_service_interface import ITileApiService -from .tile_service_interface import ITileService from .vector_service_interface import IVectorService From c42e9dbd84a6e26fdb98fcf187062f9635e4bab0 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:02 +0200 Subject: [PATCH 05/25] #361 Remove convert_pmtiles_to_bytes from IBytesService --- src/application/contracts/bytes_service_interface.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/application/contracts/bytes_service_interface.py b/src/application/contracts/bytes_service_interface.py index b2c88998..911e589f 100644 --- a/src/application/contracts/bytes_service_interface.py +++ b/src/application/contracts/bytes_service_interface.py @@ -1,5 +1,4 @@ from abc import ABC, abstractmethod -from pathlib import Path import pandas as pd import geopandas as gpd @@ -62,13 +61,3 @@ def convert_df_to_parquet_bytes(df: pd.DataFrame | gpd.GeoDataFrame) -> bytes: """ raise NotImplementedError - @staticmethod - @abstractmethod - def convert_pmtiles_to_bytes(path: Path) -> bytes: - """ - Converts a PMTiles file to a byte array. - :param path: Path to the PMTiles file. - :return: Byte array representation of the PMTiles file. - :rtype: bytes - """ - raise NotImplementedError From 0373b9e438e6925b8e95aa988b584f54cb9b2b71 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:07 +0200 Subject: [PATCH 06/25] #361 Remove TileService implementation --- .../infrastructure/services/tile_service.py | 96 ------------------- 1 file changed, 96 deletions(-) delete mode 100644 src/infra/infrastructure/services/tile_service.py diff --git a/src/infra/infrastructure/services/tile_service.py b/src/infra/infrastructure/services/tile_service.py deleted file mode 100644 index 2c9799dd..00000000 --- a/src/infra/infrastructure/services/tile_service.py +++ /dev/null @@ -1,96 +0,0 @@ -import json -import math - -from src import Config -from src.application.contracts import ITileService - - -class TileService(ITileService): - def lat_lon_to_tile( - self, - lat: float, - lon: float, - zoom: int, - bounding_box: tuple[float, float, float, float] - ) -> tuple[int, int, int]: - min_lat, min_lon, max_lat, max_lon = bounding_box - - lat = max(min_lat, min(lat, max_lat)) - lon = max(min_lon, min(lon, max_lon)) - - lat = max(min(lat, 85.05112878), -85.05112878) - - n = 2 ** zoom - - x = int((lon + 180.0) / 360.0 * n) - y = int((1.0 - math.log(math.tan(math.radians(lat)) + (1 / math.cos(math.radians(lat)))) / math.pi) / 2.0 * n) - - x = max(0, min(x, n - 1)) - y = max(0, min(y, n - 1)) - - return zoom, x, y - - def build_candidate_tiles( - self, - min_lat: float, - min_lon: float, - max_lat: float, - max_lon: float, - zoom: int - ) -> list[tuple[int, int, int]]: - _, top_left_x, top_left_y = self.lat_lon_to_tile( - lat=max_lat, - lon=min_lon, - zoom=zoom, - bounding_box=(min_lat, min_lon, max_lat, max_lon), - ) - _, bottom_right_x, bottom_right_y = self.lat_lon_to_tile( - lat=min_lat, - lon=max_lon, - zoom=zoom, - bounding_box=(min_lat, min_lon, max_lat, max_lon), - ) - - min_x = min(top_left_x, bottom_right_x) - max_x = max(top_left_x, bottom_right_x) - min_y = min(top_left_y, bottom_right_y) - max_y = max(top_left_y, bottom_right_y) - - return [ - (zoom, x, y) - for x in range(min_x, max_x + 1) - for y in range(min_y, max_y + 1) - ] - - def load_tiles(self, number_of_tiles: int) -> list[tuple[int, int, int]]: - with Config.MVT_TILES_PATH.open("r", encoding="utf-8") as f: - raw = f.read() - - if not raw or not raw.strip(): - raise ValueError(f"Tiles JSON at {Config.MVT_TILES_PATH} is empty") - - try: - data = json.loads(raw) - except json.JSONDecodeError as exc: - preview = repr(raw[:200]) - raise ValueError( - f"Failed to parse tiles JSON at {Config.MVT_TILES_PATH}: {exc}.\n" - f"File start preview (first 200 chars): {preview}\n" - "Common causes: file saved with wrong encoding/BOM (we use utf-8-sig)," - " extra characters before JSON (e.g. stray comma), or invalid JSON syntax." - ) from exc - - if not isinstance(data, list): - raise ValueError(f"Tiles JSON must be a list, got {type(data).__name__}") - - tiles: list[tuple[int, int, int]] = [] - for idx, item in enumerate(data): - if not isinstance(item, (list, tuple)) or len(item) != 3: - raise ValueError(f"Tile at index {idx} must be a 3-element list/tuple, got: {item}") - try: - z, x, y = int(item[0]), int(item[1]), int(item[2]) - except Exception as exc: - raise ValueError(f"Tile at index {idx} contains non-integer values: {item}") from exc - tiles.append((z, x, y)) - - return (tiles * ((number_of_tiles // len(tiles)) + 1))[:number_of_tiles] From de58ddf5931c373df0c8553d61540981748864c8 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:09 +0200 Subject: [PATCH 07/25] #361 Remove TileApiService implementation --- .../services/tile_api_service.py | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 src/infra/infrastructure/services/tile_api_service.py diff --git a/src/infra/infrastructure/services/tile_api_service.py b/src/infra/infrastructure/services/tile_api_service.py deleted file mode 100644 index 3e85ba82..00000000 --- a/src/infra/infrastructure/services/tile_api_service.py +++ /dev/null @@ -1,86 +0,0 @@ -from typing import Callable - -from pmtiles.reader import Reader -from requests import Session, session, RequestException - -from src import Config -from src.application.contracts import ITileApiService - - -class TileApiService(ITileApiService): - __session: Session - - def __init__(self): - self.__session = session() - - def fetch_vmt_tile(self, z: int, x: int, y: int) -> bytes | None: - try: - tile_response = self.__session.get( - f"{Config.AZURE_VMT_SERVER_URL}/tiles/{z}/{x}/{y}", - timeout=10, - headers={ - "Cache-Control": "no-cache, no-store, max-age=0", - "Pragma": "no-cache" - } - ) - except RequestException as e: - raise RuntimeError("Failed to fetch tile from VMT server") from e - - if tile_response.status_code == 404: - return None - - try: - tile_response.raise_for_status() - except RequestException as e: - raise RuntimeError("Failed to fetch tile from VMT server") from e - - if not tile_response.content: - return None - - return tile_response.content - - def fetch_pmtiles_tile(self, reader: Reader, z: int, x: int, y: int) -> bytes | None: - return reader.get(z, x, y) - - def create_pmtiles_reader(self, pmtiles_url: str) -> Reader: - return Reader(self.__http_range_source(url=pmtiles_url)) - - def __http_range_source(self, url: str) -> Callable: - def _get_bytes(offset: int, length: int) -> bytes: - end = offset + length - 1 - headers = { - "Range": f"bytes={offset}-{end}", - "Accept-Encoding": "identity", - } - r = self.__session.get(url, headers=headers, stream=False, timeout=30) - if r.status_code != 206: - if r.status_code != 200: - r.raise_for_status() - raise RuntimeError(f"Expected HTTP 206 Partial Content for range request, got {r.status_code}") - - content_range = r.headers.get("Content-Range") - if content_range: - try: - units, range_spec = content_range.split(" ", 1) - if units.strip().lower() != "bytes": - raise ValueError("Unsupported Content-Range units") - byte_range, _ = range_spec.split("/", 1) - start_str, end_str = byte_range.split("-", 1) - start = int(start_str) - end_returned = int(end_str) - except Exception as exc: - raise RuntimeError(f"Invalid Content-Range header: {content_range}") from exc - if start != offset or (end_returned - start + 1) != length: - raise RuntimeError( - f"Server returned unexpected byte range {content_range} " - f"for requested offset={offset}, length={length}" - ) - - if len(r.content) != length: - raise RuntimeError( - f"Server returned {len(r.content)} bytes, expected {length} " - f"for offset={offset}" - ) - return r.content - - return _get_bytes From b0b8751c4669b9421283f7fd8c9a6d3804f0b8ef Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:10 +0200 Subject: [PATCH 08/25] #361 Remove MVTService implementation --- .../infrastructure/services/mvt_service.py | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 src/infra/infrastructure/services/mvt_service.py diff --git a/src/infra/infrastructure/services/mvt_service.py b/src/infra/infrastructure/services/mvt_service.py deleted file mode 100644 index bc23eacd..00000000 --- a/src/infra/infrastructure/services/mvt_service.py +++ /dev/null @@ -1,50 +0,0 @@ -import asyncio -from sqlalchemy import Engine, text - -from src.application.contracts import IMVTService - - -class MVTService(IMVTService): - __db_context: Engine - - def __init__(self, db_context: Engine): - self.__db_context = db_context - - async def get_mvt_tiles(self, z: int, x: int, y: int) -> bytes | None: - query = text( - """ - WITH - tile_bounds AS ( - SELECT ST_TileEnvelope(:z, :x, :y) AS geom_3857 - ), - bounds_4326 AS ( - SELECT ST_Transform(geom_3857, 4326) AS geom - FROM tile_bounds - ), - mvtgeom AS ( - SELECT - ST_AsMVTGeom( - ST_Transform(buildings_small.geometry, 3857), - tile_bounds.geom_3857, - 4096, - 256, - true - ) AS geometry - FROM buildings_small, tile_bounds, bounds_4326 - WHERE ST_Intersects(buildings_small.geometry, bounds_4326.geom) - ) - SELECT ST_AsMVT(mvtgeom, 'buildings', 4096, 'geometry') AS tile - FROM mvtgeom - """ - ) - - def _blocking_db_call(): - with self.__db_context.connect() as conn: - result = conn.execute(query, {"z": z, "x": x, "y": y}).fetchone() - - if result is None or result[0] is None or len(result[0]) == 0: - return None - - return bytes(result[0]) - - return await asyncio.to_thread(_blocking_db_call) From 266e6a8a6e792f80a5c003ea2df754f6b36e39c4 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:12 +0200 Subject: [PATCH 09/25] #361 Remove tile re-exports from services __init__ --- src/infra/infrastructure/services/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/infra/infrastructure/services/__init__.py b/src/infra/infrastructure/services/__init__.py index 0e289049..2a69a4c5 100644 --- a/src/infra/infrastructure/services/__init__.py +++ b/src/infra/infrastructure/services/__init__.py @@ -12,13 +12,10 @@ from .file_path_service import FilePathService from .fkb_service import FKBService from .monitoring_storage_service import MonitoringStorageService -from .mvt_service import MVTService from .open_street_map_file_service import OpenStreetMapFileService from .open_street_map_service import OpenStreetMapService from .release_service import ReleaseService from .stac_io_service import StacIOService from .stac_service import StacService from .test_dataset_service import TestDatasetService -from .tile_api_service import TileApiService -from .tile_service import TileService from .vector_service import VectorService From 17a3f9e98c198384ab2ad5b9eee94cd2ec715e06 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:12 +0200 Subject: [PATCH 10/25] #361 Remove convert_pmtiles_to_bytes from BytesService --- .../infrastructure/services/bytes_service.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/infra/infrastructure/services/bytes_service.py b/src/infra/infrastructure/services/bytes_service.py index 8be22a12..a9903486 100644 --- a/src/infra/infrastructure/services/bytes_service.py +++ b/src/infra/infrastructure/services/bytes_service.py @@ -1,5 +1,4 @@ from io import BytesIO -from pathlib import Path import geopandas as gpd import pandas as pd @@ -50,18 +49,3 @@ def convert_df_to_parquet_bytes(df: pd.DataFrame | gpd.GeoDataFrame) -> bytes: buffer.seek(0) return buffer.read() - @staticmethod - def convert_pmtiles_to_bytes(path: Path) -> bytes: - if not path.exists(): - logger.error("PMTiles file not found: %s", path) - raise FileNotFoundError(f"PMTiles file not found: {path}") - - try: - with path.open("rb") as f: - data = f.read() - if not data: - logger.warning("PMTiles file %s is empty.", path) - return data - except Exception as e: - logger.exception("Failed to read PMTiles file %s: %s", path, e) - raise From 0de92b1b1258ca0b2c359e930de838a1a37e89f9 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:18 +0200 Subject: [PATCH 11/25] #361 Remove tile/MVT service wiring from DI container --- src/infra/infrastructure/containers.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/infra/infrastructure/containers.py b/src/infra/infrastructure/containers.py index 86eeae26..dff47d62 100644 --- a/src/infra/infrastructure/containers.py +++ b/src/infra/infrastructure/containers.py @@ -4,7 +4,7 @@ from src.infra.infrastructure.services import ( BlobStorageService, OpenStreetMapService, OpenStreetMapFileService, FilePathService, ReleaseService, BytesService, CountyService, VectorService, StacService, StacIOService, FKBService, ConflationService, - TestDatasetService, DatasetSynthesisService, MonitoringStorageService, MVTService, TileApiService, TileService, + TestDatasetService, DatasetSynthesisService, MonitoringStorageService, AzureCostService, BenchmarkConfigurationService, AzureMetricService, AzurePricingService, BenchmarkService, DatabricksService ) @@ -98,19 +98,6 @@ class Containers(containers.DeclarativeContainer): duckdb_context=duckdb_context ) - mvt_service = providers.Singleton( - MVTService, - db_context=postgres_context - ) - - tile_api_service = providers.Singleton( - TileApiService - ) - - tile_service = providers.Singleton( - TileService - ) - benchmark_configuration_service = providers.Singleton( BenchmarkConfigurationService ) From 71ae5d8cc20159553bd1ccf121244dac86fa8f82 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:19 +0200 Subject: [PATCH 12/25] #361 Remove tile_server.py and endpoints directory --- src/presentation/endpoints/__init__.py | 0 src/presentation/endpoints/tile_server.py | 62 ----------------------- 2 files changed, 62 deletions(-) delete mode 100644 src/presentation/endpoints/__init__.py delete mode 100644 src/presentation/endpoints/tile_server.py diff --git a/src/presentation/endpoints/__init__.py b/src/presentation/endpoints/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/presentation/endpoints/tile_server.py b/src/presentation/endpoints/tile_server.py deleted file mode 100644 index 3757aa7e..00000000 --- a/src/presentation/endpoints/tile_server.py +++ /dev/null @@ -1,62 +0,0 @@ -from contextlib import asynccontextmanager - -from dependency_injector.wiring import Provide, inject -from fastapi import FastAPI, HTTPException, Response -from starlette.middleware.cors import CORSMiddleware - -from src.application.contracts import IMVTService -from src.infra.infrastructure import Containers -from src.presentation.configuration import initialize_dependencies - - -@inject -async def _db_call(z: int, x: int, y: int, mvt_service: IMVTService = Provide[Containers.mvt_service]): - return await mvt_service.get_mvt_tiles(z=z, x=x, y=y) - - -@asynccontextmanager -async def lifespan(_app: FastAPI): - """FastAPI lifespan context that initializes the DI container before serving requests.""" - initialize_dependencies(run_id="not-needed", benchmark_run=1) - yield - - -app = FastAPI(lifespan=lifespan) -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=False, - allow_methods=["*"], - allow_headers=["*"], -) - - -@app.get("/tiles/{z}/{x}/{y}") -async def get_tiles(z: int, x: int, y: int): - """ - HTTP GET ``/tiles/{z}/{x}/{y}``. Returns the buildings MVT tile as - ``application/x-protobuf`` bytes. Responds 400 for zoom levels outside [0, 22] - and 404 when no features intersect the tile. Caching headers are disabled. - :param z: Tile zoom level (0-22). - :param x: Tile X coordinate. - :param y: Tile Y coordinate. - :return: FastAPI Response carrying the MVT tile bytes. - :rtype: Response - """ - if z < 0 or z > 22: - raise HTTPException(status_code=400, detail="Invalid zoom") - - tile = await _db_call(z, x, y) - if not tile: - raise HTTPException(status_code=404, detail="Tile not found") - - return Response( - content=tile, - media_type="application/x-protobuf", - headers={ - "Content-Encoding": "identity", - "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0, s-maxage=0", - "Pragma": "no-cache", - "Expires": "0", - } - ) From 997a0b972c847d2860069f640f284220df50dd0c Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:20 +0200 Subject: [PATCH 13/25] #361 Remove tile_server from DI wire modules --- src/presentation/configuration/app_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/presentation/configuration/app_config.py b/src/presentation/configuration/app_config.py index 1574277c..7bc253ee 100644 --- a/src/presentation/configuration/app_config.py +++ b/src/presentation/configuration/app_config.py @@ -53,8 +53,6 @@ def initialize_dependencies( "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_8_nodes", "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_12_nodes", "src.presentation.entrypoints.national_scale_spatial_join_databricks_partitioned_16_nodes", - "src.presentation.entrypoints.setup_benchmarking_framework", - - "src.presentation.endpoints.tile_server" + "src.presentation.entrypoints.setup_benchmarking_framework" ] ) From 700f79bf4c89cf9a8295a897b3aa4c174b2de73e Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:21 +0200 Subject: [PATCH 14/25] #361 Remove PMTiles/MVT setup functions from benchmarking framework --- .../setup_benchmarking_framework.py | 221 +----------------- 1 file changed, 1 insertion(+), 220 deletions(-) diff --git a/src/presentation/entrypoints/setup_benchmarking_framework.py b/src/presentation/entrypoints/setup_benchmarking_framework.py index da370f9d..d1f2beca 100644 --- a/src/presentation/entrypoints/setup_benchmarking_framework.py +++ b/src/presentation/entrypoints/setup_benchmarking_framework.py @@ -1,7 +1,4 @@ -import json -import subprocess - -import geopandas as gpd +import geopandas as gpd from osgeo import ogr from pyproj import CRS from dependency_injector.wiring import Provide, inject @@ -15,9 +12,6 @@ from src.application.contracts import ( IFilePathService, IBlobStorageService, - IBytesService, - ITileService, - ITileApiService, ITestDatasetService, IDatasetSynthesisService, IBenchmarkService, @@ -223,219 +217,6 @@ def _seed_postgres_for_size( ) -@inject -def _create_pmtiles( - release: str | None = None, - duckdb_context: DuckDBPyConnection = Provide[Containers.duckdb_context], - file_path_service: IFilePathService = Provide[Containers.file_path_service], - blob_storage_service: IBlobStorageService = Provide[ - Containers.blob_storage_service - ], - bytes_service: IBytesService = Provide[Containers.bytes_service], -) -> None: - path = file_path_service.create_release_virtual_filesystem_path( - storage_scheme="az", - release=release or Config.BENCHMARK_DOPPA_DATA_RELEASE, - container=StorageContainer.DATA, - theme=Theme.BUILDINGS, - dataset_size=DatasetSize.SMALL, - region="*", - file_name="*.parquet", - ) - - Config.BUILDINGS_GEOJSONL_FILE.parent.mkdir(parents=True, exist_ok=True) - Config.BUILDINGS_PMTILES_FILE.parent.mkdir(parents=True, exist_ok=True) - - logger.info("Fetching buildings as GeoJSONL file.") - - duckdb_context.execute(f""" - COPY ( - SELECT - * EXCLUDE (geometry, bbox), - ST_XMax(geometry) AS bbox_maxx, - ST_YMax(geometry) AS bbox_maxy, - ST_XMin(geometry) AS bbox_minx, - ST_YMin(geometry) AS bbox_miny, - geometry - FROM read_parquet('{path}') - ) - TO '{Config.BUILDINGS_GEOJSONL_FILE.as_posix()}' - WITH ( - FORMAT GDAL, - DRIVER 'GeoJSONSeq' - ); - """) - - logger.info(f"Saved buildings to '{Config.BUILDINGS_GEOJSONL_FILE}'") - - cmd = [ - "tippecanoe", - "-o", - Config.BUILDINGS_PMTILES_FILE.as_posix(), - "-zg", - "--drop-densest-as-needed", - "--coalesce", - "--read-parallel", - "-l", - "buildings", - Config.BUILDINGS_GEOJSONL_FILE.as_posix(), - ] - - logger.info("Running tippecanoe to generate PMTiles (this may take a while)...") - result = subprocess.run(cmd, capture_output=True, text=True) - - if result.returncode != 0: - raise RuntimeError( - f"tippecanoe failed with exit code {result.returncode}:\n" - f"STDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}" - ) - - logger.info(f"PMTiles saved to '{Config.BUILDINGS_PMTILES_FILE}'") - logger.info("Uploading PMTiles to blob storage.") - - pmtiles_bytes = bytes_service.convert_pmtiles_to_bytes( - Config.BUILDINGS_PMTILES_FILE - ) - blob_storage_service.upload_file( - container_name=StorageContainer.TILES, - blob_name=Config.BUILDINGS_PMTILES_FILE.name, - data=pmtiles_bytes, - ) - - logger.info(f"Uploaded PMTiles to container '{StorageContainer.TILES.value}'") - - -@inject -def _create_mvt( - release: str | None = None, - duckdb_context: DuckDBPyConnection = Provide[Containers.duckdb_context], - file_path_service: IFilePathService = Provide[Containers.file_path_service], - blob_storage_service: IBlobStorageService = Provide[ - Containers.blob_storage_service - ], -) -> None: - path = file_path_service.create_release_virtual_filesystem_path( - storage_scheme="az", - release=release or Config.BENCHMARK_DOPPA_DATA_RELEASE, - container=StorageContainer.DATA, - theme=Theme.BUILDINGS, - dataset_size=DatasetSize.SMALL, - region="*", - file_name="*.parquet", - ) - - Config.BUILDINGS_GEOJSONL_FILE.parent.mkdir(parents=True, exist_ok=True) - Config.BUILDINGS_MVT_DIR.parent.mkdir(parents=True, exist_ok=True) - - if not Config.BUILDINGS_GEOJSONL_FILE.exists(): - logger.info("Fetching buildings as GeoJSONL file for MVT generation.") - - duckdb_context.execute(f""" - COPY ( - SELECT - * EXCLUDE (geometry, bbox), - ST_XMax(geometry) AS bbox_maxx, - ST_YMax(geometry) AS bbox_maxy, - ST_XMin(geometry) AS bbox_minx, - ST_YMin(geometry) AS bbox_miny, - geometry - FROM read_parquet('{path}') - ) - TO '{Config.BUILDINGS_GEOJSONL_FILE.as_posix()}' - WITH ( - FORMAT GDAL, - DRIVER 'GeoJSONSeq' - ); - """) - - logger.info(f"Saved buildings to '{Config.BUILDINGS_GEOJSONL_FILE}'") - else: - logger.info( - f"GeoJSONL file already exists at '{Config.BUILDINGS_GEOJSONL_FILE}', skipping creation." - ) - - cmd = [ - "tippecanoe", - "-e", - Config.BUILDINGS_MVT_DIR.as_posix(), - "-zg", - "--drop-densest-as-needed", - "--coalesce", - "--read-parallel", - "--no-tile-compression", - "--force", - "-l", - "buildings", - Config.BUILDINGS_GEOJSONL_FILE.as_posix(), - ] - - logger.info("Running tippecanoe to generate MVT tiles (this may take a while)...") - result = subprocess.run(cmd, capture_output=True, text=True) - - if result.returncode != 0: - raise RuntimeError( - f"tippecanoe failed with exit code {result.returncode}:\n" - f"STDOUT:\n{result.stdout}\n\nSTDERR:\n{result.stderr}" - ) - - logger.info(f"MVT tiles saved to '{Config.BUILDINGS_MVT_DIR}'") - logger.info("Uploading MVT tiles to blob storage.") - - mvt_dir = Config.BUILDINGS_MVT_DIR - tile_count = 0 - - for tile_file in mvt_dir.rglob("*.pbf"): - relative_path = tile_file.relative_to(mvt_dir) - blob_name = f"mvt/{relative_path.as_posix()}" - - tile_bytes = tile_file.read_bytes() - blob_storage_service.upload_file( - container_name=StorageContainer.TILES, - blob_name=blob_name, - data=tile_bytes, - ) - tile_count += 1 - - logger.info( - f"Uploaded {tile_count} MVT tiles to container '{StorageContainer.TILES.value}' under 'mvt/' prefix." - ) - - -@inject -def _generate_tiles_file( - tile_service: ITileService = Provide[Containers.tile_service], - tile_api_service: ITileApiService = Provide[Containers.tile_api_service], -) -> None: - TILE_ZOOM: int = 13 - - min_lat, min_lon, max_lat, max_lon = Config.BUILDINGS_SPATIAL_EXTENT - candidate_tiles = tile_service.build_candidate_tiles( - min_lat=min_lat, - min_lon=min_lon, - max_lat=max_lat, - max_lon=max_lon, - zoom=TILE_ZOOM, - ) - - logger.info(f"Created {len(candidate_tiles)} candidate tiles") - logger.info("Finding candidate tiles with data...") - - existing_tiles: list[tuple[int, int, int]] = [] - for candidate_tile in candidate_tiles: - z, x, y = candidate_tile - if tile_api_service.fetch_vmt_tile(z=z, x=x, y=y) is not None: - existing_tiles.append(candidate_tile) - - logger.info( - f"Found {len(existing_tiles)} tiles with data out of {len(candidate_tiles)} candidates" - ) - - with open(Config.MVT_TILES_PATH, "w", encoding="utf-8") as f: - json.dump([list(tile) for tile in existing_tiles], f) - - logger.info(f"Tiles file saved to '{Config.MVT_TILES_PATH}'") - - @inject def _create_shapefile_copy( release: str | None = None, From f3bc6813ccd4bdf628a0bb6b63469aa066c500a0 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:22 +0200 Subject: [PATCH 15/25] #361 Remove tile-related config vars from Config --- src/config.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/config.py b/src/config.py index b2c7aad1..8ae6379e 100644 --- a/src/config.py +++ b/src/config.py @@ -40,7 +40,6 @@ class Config: "AZURE_BLOB_STORAGE_CONNECTION_STRING" ) AZURE_BLOB_STORAGE_MAX_CONCURRENCY: int = 1 - AZURE_VMT_SERVER_URL: str = "https://doppa-vmt.azurewebsites.net" AZURE_METRICS_REGIONAL_ENDPOINT: str = ( f"https://{AZURE_RESOURCE_LOCATION}.metrics.monitor.azure.com" ) @@ -59,11 +58,6 @@ class Config: LOG_DIR: Path = ROOT_DIR / f"logs" BUILDINGS_SHAPEFILE: Path = ROOT_DIR / "resources" / "buildings.shp" BUILDINGS_PARQUET_FILE: Path = ROOT_DIR / "resources" / "buildings.parquet" - BUILDINGS_GEOJSONL_FILE: Path = ROOT_DIR / "resources" / "buildings.geojsonl" - BUILDINGS_PMTILES_FILE: Path = ROOT_DIR / "resources" / "buildings.pmtiles" - BUILDINGS_MVT_DIR: Path = ROOT_DIR / "resources" / "buildings_mvt" - MVT_TILES_PATH: Path = ROOT_DIR / "resources" / "tiles.json" - # LOGGING LOGGING_LEVEL: int = logging.INFO LOG_FILE: Path = LOG_DIR / f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" @@ -84,8 +78,6 @@ class Config: KNN_SEARCH_K: int = 10 # NTNU Hovedbygget (Main Building) — large footprint guaranteed in the buildings dataset POINT_IN_POLYGON_PROBE_WGS84: tuple[float, float] = (10.4044, 63.4187) - VECTOR_TILES_100K_TOTAL_REQUESTS: int = 100_000 - # STAC STAC_LICENSE = "CC-BY-4.0" STAC_STORAGE_CONTAINER = "https://doppabs.blob.core.windows.net/stac" From c2439e00e29268b165e0f9af19c3251a7a4fe993 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:27 +0200 Subject: [PATCH 16/25] #361 Remove VECTOR_TILE enum members from BenchmarkIteration --- src/domain/enums/benchmark_iteration.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/domain/enums/benchmark_iteration.py b/src/domain/enums/benchmark_iteration.py index 17dcc1f7..d03f57c9 100644 --- a/src/domain/enums/benchmark_iteration.py +++ b/src/domain/enums/benchmark_iteration.py @@ -14,7 +14,4 @@ class BenchmarkIteration(Enum): ORDERED_RANGE_QUERY = 1500 POINT_IN_POLYGON_LOOKUP = 2500 SPATIAL_AGGREGATION_GRID = 100 - VECTOR_TILE_100K = 2 - VECTOR_TILE_SINGLE_TILE = 1500 - FALLBACK = Config.BENCHMARK_ITERATIONS From 0c82e7d3056b8b6e84c812143e87a0d35dc08c27 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:29 +0200 Subject: [PATCH 17/25] #361 Remove TILES from StorageContainer enum --- src/domain/enums/storage_container.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/domain/enums/storage_container.py b/src/domain/enums/storage_container.py index 73f28f0e..02637d4b 100644 --- a/src/domain/enums/storage_container.py +++ b/src/domain/enums/storage_container.py @@ -12,4 +12,3 @@ class StorageContainer(Enum): OPEN_STREET_MAP = "open_street_map" FKB = "fkb" BENCHMARKS = os.getenv("AZURE_BLOB_STORAGE_BENCHMARK_CONTAINER") - TILES = "tiles" From 89375dcf03008f577abd6d77d5e46b559da93678 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:31 +0200 Subject: [PATCH 18/25] #361 Remove Api.Dockerfile --- .docker/Api.Dockerfile | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .docker/Api.Dockerfile diff --git a/.docker/Api.Dockerfile b/.docker/Api.Dockerfile deleted file mode 100644 index 775391cf..00000000 --- a/.docker/Api.Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.11-slim -WORKDIR /app -COPY requirements.txt /app/ -RUN apt-get update && apt-get install -y --fix-missing \ - libexpat1 \ - libgdal-dev \ - g++ \ - && pip install --no-cache-dir gdal==$(gdal-config --version) \ - && apt-get purge -y g++ \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir -r requirements.txt -COPY . /app - -EXPOSE 8000 \ No newline at end of file From ef565e71e6c855fe7b5eb42a868f2195a9a5c2b9 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:32 +0200 Subject: [PATCH 19/25] #361 Remove vmt-api-server from docker-compose --- docker-compose.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2fce66e5..612703ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -217,14 +217,3 @@ services: dockerfile: .docker/Query.Dockerfile image: national-scale-spatial-join-databricks-partitioned-16-nodes:latest command: python benchmark_runner.py --script-id national-scale-spatial-join-databricks-partitioned-16-nodes --benchmark-run 1 --run-id ABCDEF - - vmt-api-server: - env_file: - - .env - build: - context: . - dockerfile: .docker/Api.Dockerfile - ports: - - "8000:8000" - image: vmt-api-server:latest - command: uvicorn src.presentation.endpoints.tile_server:app --host 0.0.0.0 --port 8000 From af2aaec0071d24f26e7068c33f08c2949127a89b Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:34 +0200 Subject: [PATCH 20/25] #361 Remove publish-api workflow --- .github/workflows/publish-api.yml | 132 ------------------------------ 1 file changed, 132 deletions(-) delete mode 100644 .github/workflows/publish-api.yml diff --git a/.github/workflows/publish-api.yml b/.github/workflows/publish-api.yml deleted file mode 100644 index 6bb5140d..00000000 --- a/.github/workflows/publish-api.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Publish APIs - -on: - pull_request: - types: [ closed ] - workflow_dispatch: - - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -jobs: - detect-changes: - name: Detect changed paths - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true - permissions: - contents: read - pull-requests: read - outputs: - api: ${{ steps.filter.outputs.api || (github.event_name == 'workflow_dispatch' && 'true') }} - steps: - - uses: actions/checkout@v4 - - - uses: dorny/paths-filter@v3 - id: filter - if: github.event_name == 'pull_request' - with: - filters: | - api: - - 'src/**' - - '!src/presentation/entrypoints/**' - - '!src/presentation/databricks/**' - - '.docker/Api.Dockerfile' - - 'requirements.txt' - - 'docker-compose.yml' - - '.github/workflows/publish-api.yml' - - build-and-push-api-images-to-acr: - name: Build & Push API Images to ACR - needs: detect-changes - if: needs.detect-changes.outputs.api == 'true' - runs-on: ubuntu-latest - strategy: - matrix: - include: - - service: vmt-api-server - display_name: VMT API Server - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Azure Login - uses: azure/login@v2 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - - name: Log in to ACR - run: az acr login --name ${{ vars.ACR_NAME }} - - - name: Build ${{ matrix.display_name }} from docker-compose - run: docker compose build ${{ matrix.service }} - - - name: Tag ${{ matrix.display_name }} as latest - run: docker tag ${{ matrix.service }}:latest ${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.service }}:latest - - - name: Tag ${{ matrix.display_name }} with commit SHA - run: docker tag ${{ matrix.service }}:latest ${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.service }}:${{ github.sha }} - - - name: Push ${{ matrix.display_name }} (latest) - run: docker push ${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.service }}:latest - - - name: Push ${{ matrix.display_name }} (commit SHA) - run: docker push ${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.service }}:${{ github.sha }} - - deploy-vmt-api: - name: Deploy VMT API to Azure Web App - runs-on: ubuntu-latest - needs: build-and-push-api-images-to-acr - strategy: - matrix: - include: - - service: vmt-api-server - display_name: VMT API Server - webapp_name: doppa-vmt - - steps: - - name: Azure Login - uses: azure/login@v2 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - - name: Configure app settings - uses: azure/cli@v2 - env: - WEBAPP_NAME: ${{ matrix.webapp_name }} - RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} - POSTGRES_USERNAME: ${{ secrets.POSTGRES_USERNAME }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - POSTGRES_SERVER_NAME: ${{ vars.POSTGRES_SERVER_NAME }} - BLOB_CONN_STRING: ${{ secrets.AZURE_BLOB_STORAGE_CONNECTION_STRING }} - BLOB_BENCHMARK: ${{ vars.AZURE_BLOB_STORAGE_BENCHMARK_CONTAINER }} - BLOB_METADATA: ${{ vars.AZURE_BLOB_STORAGE_METADATA_CONTAINER }} - with: - azcliversion: latest - inlineScript: | - az webapp config appsettings set \ - --name "$WEBAPP_NAME" \ - --resource-group "$RESOURCE_GROUP" \ - --settings \ - POSTGRES_USERNAME="$POSTGRES_USERNAME" \ - POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ - POSTGRES_SERVER_NAME="$POSTGRES_SERVER_NAME" \ - AZURE_BLOB_STORAGE_CONNECTION_STRING="$BLOB_CONN_STRING" \ - AZURE_BLOB_STORAGE_BENCHMARK_CONTAINER="$BLOB_BENCHMARK" \ - AZURE_BLOB_STORAGE_METADATA_CONTAINER="$BLOB_METADATA" - - - name: Deploy ${{ matrix.display_name }} - uses: azure/webapps-deploy@v3 - with: - app-name: ${{ matrix.webapp_name }} - images: ${{ vars.ACR_LOGIN_SERVER }}/${{ matrix.service }}:latest \ No newline at end of file From 2de48cd235c32e23252f93c4c183c9f6c1987f8d Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:40 +0200 Subject: [PATCH 21/25] #361 Remove VMT API build job from PR tests workflow --- .github/workflows/pull-request-tests.yml | 26 ------------------------ 1 file changed, 26 deletions(-) diff --git a/.github/workflows/pull-request-tests.yml b/.github/workflows/pull-request-tests.yml index b6f61e6c..f651ad2b 100644 --- a/.github/workflows/pull-request-tests.yml +++ b/.github/workflows/pull-request-tests.yml @@ -31,7 +31,6 @@ jobs: pull-requests: read outputs: orchestrator: ${{ steps.filter.outputs.orchestrator }} - api: ${{ steps.filter.outputs.api }} benchmarks: ${{ steps.filter.outputs.benchmarks }} steps: - uses: actions/checkout@v4 @@ -57,13 +56,6 @@ jobs: - '.docker/Setup.Dockerfile' - 'requirements.txt' - 'docker-compose.yml' - api: - - 'src/**' - - '!src/presentation/entrypoints/**' - - '!src/presentation/databricks/**' - - '.docker/Api.Dockerfile' - - 'requirements.txt' - - 'docker-compose.yml' compile: name: Check Python syntax @@ -109,24 +101,6 @@ jobs: - name: Build Container Orchestrator from docker-compose run: docker compose build container-orchestrator - build-api-image: - name: Build VMT API Server - runs-on: ubuntu-latest - needs: - - compile - - detect-changes - if: needs.detect-changes.outputs.api == 'true' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build VMT API Server from docker-compose - run: docker compose build vmt-api-server - build-benchmark-images: name: Build ${{ matrix.display_name }} runs-on: ubuntu-latest From c016ae0a5e45992de128f897a398d6ba1a879277 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:42 +0200 Subject: [PATCH 22/25] #361 Remove tile-related dependencies from requirements --- requirements.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index ead51e16..0999347d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,6 @@ dependency-injector==4.48.2 dotenv==0.9.9 duckdb==1.4.0 executing==2.2.1 -fastapi==0.135.1 fastjsonschema==2.21.2 fiona==1.10.1 folium==0.20.0 @@ -89,7 +88,6 @@ lark==1.3.0 MarkupSafe==3.0.3 matplotlib==3.10.6 matplotlib-inline==0.1.7 -mercantile==1.2.1 mistune==3.1.4 msal==1.34.0 msal-extensions==1.3.1 @@ -111,7 +109,6 @@ parso==0.8.5 pexpect==4.9.0 pillow==11.3.0 platformdirs==4.4.0 -pmtiles==3.7.0 prometheus_client==0.23.1 prompt_toolkit==3.0.52 propcache==0.4.1 @@ -152,7 +149,6 @@ sniffio==1.3.1 soupsieve==2.8 SQLAlchemy==2.0.47 stack-data==0.6.3 -starlette==0.52.1 terminado==0.18.1 tinycss2==1.4.0 tornado==6.5.2 @@ -164,7 +160,6 @@ typing_extensions==4.15.0 tzdata==2025.2 uri-template==1.3.0 urllib3==2.5.0 -uvicorn==0.41.0 viztracer==1.1.1 watchfiles==1.1.1 wcwidth==0.2.14 From afc845a222ebc56c454fbc7e6bdffecb68c48eb5 Mon Sep 17 00:00:00 2001 From: Jathavaan Shankarr Date: Wed, 27 May 2026 10:54:43 +0200 Subject: [PATCH 23/25] #361 Remove tile/VMT references from README --- README.md | 39 ++++----------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 39eee7d8..c6722dcc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ doppa is a reproducible benchmarking framework for evaluating traditional geospatial query stacks (PostGIS, shapefiles) against cloud-native geospatial (CNG) alternatives (DuckDB over GeoParquet in -blob storage, PMTiles/MVT vector tiles, and Apache Sedona on Databricks) across a range of real-world +blob storage and Apache Sedona on Databricks) across a range of real-world spatial query patterns: point-in-polygon lookups, k-nearest-neighbour search, bounding-box filtering, and a national-scale spatial join. @@ -60,7 +60,7 @@ format internals to client-observed cost is measured end to end. **Cloud-native vector formats vs. traditional formats on cloud storage.** Empirical comparisons in the literature (Holmes 2023; Flatgeobuf 2024) measure write times and file sizes on local disk and do not place cloud-native and traditional formats side by side on cloud storage. doppa benchmarks GeoParquet over Azure Blob Storage (via DuckDB) -against PostGIS on Azure Database for PostgreSQL, and PMTiles against WMS-style vector tiles, across the active +against PostGIS on Azure Database for PostgreSQL, across the active catalog of query patterns: point-in-polygon lookups, k-nearest-neighbour search, bounding-box filtering, and a national-scale spatial join. The local-Shapefile entrypoints sit on the side as a laptop-workflow reference, with the Shapefile downloaded ahead of the timed scope to emulate that workflow rather than to bench the format on cloud @@ -167,8 +167,6 @@ to the elapsed-time distribution. | PostGIS | Single-node, managed service | Azure Database for PostgreSQL Flexible Server | | GeoPandas + Shapefile | Single-node, local-disk baseline | Shapefile pre-downloaded to the container before the timed scope | | Apache Sedona | Distributed | Azure Databricks, 2 / 4 / 8 / 12 / 16 `Standard_D4s_v3` workers, reading GeoParquet via ABFS | -| PMTiles | Cloud-native vector tiles | PMTiles archive in blob storage, accessed via HTTP range reads | -| WMS-style vector tiles | Traditional vector tiles | `doppa-vmt` web app for containers, tiles assembled on demand | DuckDB and PostGIS each run inside an Azure Container Instance with 4 vCPU and 16 GB RAM, so CPU and memory baselines match between the single-node engines. @@ -348,7 +346,7 @@ so. #### Resource naming The resource names used throughout this section (`doppa`, `doppabs`, `doppaacr`, `doppa-uami`, -`doppa-db`, `doppa-vmt`, `doppa-databricks`) are baked into source and configuration. Keep them +`doppa-db`, `doppa-databricks`) are baked into source and configuration. Keep them as-is for the simplest setup; this is also what the thesis deployment uses, so reproducing the published results requires these exact names. @@ -356,9 +354,8 @@ If you need to rename a resource, the following references must be updated toget | Location | What is hardcoded | |-------------------------------------|--------------------------------------------------------------------------------| -| `src/config.py` | Default values for resource group, blob URL/account, VMT URL, STAC container | +| `src/config.py` | Default values for resource group, blob URL/account, STAC container | | `benchmarks.yml` | ACR image references (`doppaacr.azurecr.io/:latest`) for every benchmark | -| `.github/workflows/publish-api.yml` | `webapp_name: doppa-vmt` | `src/config.py` defaults can also be overridden via the corresponding environment variables (see [Local development](#local-development) and [GitHub Actions](#github-actions)) without @@ -464,34 +461,6 @@ same setting change the following: - `effective_cache_size`: `6291456` - `work_mem`: `65536` -#### Web app for containers - -Create -a [web app for containers](https://portal.azure.com/#view/Microsoft_Azure_Marketplace/GalleryItemDetailsBladeNopdl/id/Microsoft.AppSvcLinux/selectionMode~/false/resourceGroupId//resourceGroupLocation//dontDiscardJourney~/false/selectedMenuId/home/launchingContext~/%7B%22galleryItemId%22%3A%22Microsoft.AppSvcLinux%22%2C%22source%22%3A%5B%22GalleryFeaturedMenuItemPart%22%2C%22VirtualizedTileDetails%22%5D%2C%22menuItemId%22%3A%22home%22%2C%22subMenuItemId%22%3A%22Search%20results%22%2C%22telemetryId%22%3A%22135c4e97-6a92-446e-aa0a-3f2201ddfdb1%22%7D/searchTelemetryId/c154ee0a-06d6-49e4-a17f-3820937e6335) -The process is the same for each of the following API servers: - -- `doppa-vmt` - -Under *Basics*: - -- Resource group: `doppa` -- Name: `` -- Publish: `Container` -- Operating system: `Linux` -- Pricing plan: `Premium V4 P0V4` - -Under *Container*: - -- Image source: `Azure Container Registry` -- Registry: `doppaacr` -- Authentication: `Managed identity` -- Identity: `doppa-uami` -- Image: `