From ffe06f3d98439aeb6d1d6998a26bb7b2a6c09fe0 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 10:39:54 +0200 Subject: [PATCH 01/58] pipeline v1 --- .buildkite/pipeline.py | 208 ++++++++++++++++++++++++++++++++++++ .buildkite/requirements.txt | 2 + .buildkite/steps/_test.yml | 39 +++++++ .gitmodules | 3 + docker/docker-compose.yml | 10 ++ 5 files changed, 262 insertions(+) create mode 100644 .buildkite/pipeline.py create mode 100644 .buildkite/requirements.txt create mode 100644 .buildkite/steps/_test.yml create mode 100644 docker/docker-compose.yml diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py new file mode 100644 index 000000000..ac1927797 --- /dev/null +++ b/.buildkite/pipeline.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Buildkite dynamic pipeline generator for qdb-api-python. + +Step templates in steps/*.yml define nearly-complete Buildkite steps with +{placeholder} variables. This script loads them, substitutes variables, and +overlays environment variables and the Docker plugin per platform. + +Usage: + python3 pipeline.py # emit pipeline YAML to stdout + python3 pipeline.py check # validate without emitting +""" +from __future__ import annotations + +import dataclasses +import sys +from pathlib import Path +import os + +from buildkite_sdk import CommandStep, Pipeline + +sys.path.insert(0, str(Path(__file__).parent / "tools")) +from qdb_pipeline import ( + Platform, + apply_docker, + load_template, + merge_env, + select_platforms, + validate_pipeline, +) # noqa: E402 + +STEPS_DIR = Path(__file__).parent / "steps" + +# Quasardb-specific toolchain overlays on top of shared infrastructure platforms. +_LINUX = dict( + # c_compiler="/usr/local/gcc13/bin/gcc", + # cxx_compiler="/usr/local/gcc13/bin/g++", + # asm_compiler="/usr/local/bin/yasm", + # ccache="/usr/local/bin/ccache", + docker_image="bureau14/builder:rhel7", + # docker_volumes=("/var/lib/ccache:/var/lib/ccache",), +) +_WIN = dict( + asm_compiler="C:\\yasm\\yasm-1.3.0-win64.exe", + # ccache="C:\\ccache\\ccache.exe", +) +_MACOS = dict( + c_compiler="/usr/local/clang21/bin/clang", + cxx_compiler="/usr/local/clang21/bin/clang++", + # ccache="/opt/local/bin/ccache", +) +_OS_OVERLAY = {"linux": _LINUX, "windows": _WIN, "macos": _MACOS} +PLATFORMS: list[Platform] = [ + dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) + for p in select_platforms( + "linux-amd64-core2", + # "linux-amd64-haswell", + # "linux-aarch64", + # "windows-amd64-core2", + # "windows-amd64-haswell", + # "macos-aarch64", + ) +] + +BUILD_TYPES = ["Release"] # Temp + +PYTHON_VERSIONS = [ + # "3.9", + # "3.10", + # "3.11", + # "3.12", + # "3.13", + "3.14", +] + +# Environment variable layering: global → step → os → os+step → platform compilers. +GLOBAL_ENV: dict[str, str] = { + # "CCACHE_COMPILERCHECK": "%compiler% -dumpmachine; %compiler% -dumpversion", + # "CCACHE_REMOTE_STORAGE": ( + # "http://ccache.cicd.intra.quasar.ai/" + # "|connect-timeout=1000|operation-timeout=30000|keep-alive=true" + # ), + # "CCACHE_RESHARE": "true", + "AWS_DEFAULT_REGION": "eu-west-1", + "QDB_ENCRYPT_TRAFFIC": "1", + "PYTHON_EXECUTABLE": "/usr/bin/python3", + "PYTHON_CMD": "python3", +} + +STEP_ENV: dict[str, dict[str, str]] = { + # "build_docker": {}, + # "start_services": { + # }, + # "test": { + # }, + # "stop_services": {}, +} + +OS_ENV: dict[str, dict[str, str]] = { + # "linux": {"CCACHE_DIR": "/var/lib/ccache"}, + # "freebsd": {}, + # "macos": {}, + # "windows": {"CCACHE_COMPILERCHECK": ""}, +} + +OS_STEP_ENV: dict[str, dict[str, str]] = { + # "linux/build": {"QDB_ENABLE_API_DOCS": "ON"}, +} + +CPU_ENV: dict[str, dict[str, str]] = { + # "core2": {"QDB_CPU_ARCHITECTURE_CORE2": "ON"}, +} + + +def _env(p: Platform, step_name: str, build_type: str) -> dict[str, str]: + """Compose the full environment dict for one step.""" + return merge_env( + GLOBAL_ENV, + STEP_ENV.get(step_name, {}), + OS_ENV.get(p.os, {}), + OS_STEP_ENV.get(f"{p.os}/{step_name}", {}), + CPU_ENV.get(p.cpu, {}), + {"CMAKE_BUILD_TYPE": build_type}, + platform=p, + ) + +def _get_git_ref() -> str: + branch = os.environ.get("BUILDKITE_BRANCH") + tag = os.environ.get("BUILDKITE_TAG") + if not branch and not tag: + raise ValueError( + "BUILDKITE_BRANCH and BUILDKITE_TAG are both empty — are we running inside a Buildkite job?" + ) + + ref = f"refs/tags/{tag}" if tag else f"refs/heads/{branch}" + + return ref + + +def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str]) -> None: + """ + Goes through list of plugins and fills in defaults for qdb-artifacts plugin if present. + Doesn't overwrite existing keys, only fills in missing ones from provided dict. + """ + plugins = step.get("plugins", {}) + + for plugin_dict in plugins: + plugin_name = list(plugin_dict.keys())[0] + if plugin_name.startswith("bureau14/qdb-artifacts#"): + plugin_config = plugin_dict[plugin_name] + + for config in plugin_config.values(): + for key in vars: + if key not in config: + config[key] = vars[key] + +def generate_pipeline() -> Pipeline: + """Load templates, expand across platforms × build_types, overlay env and docker.""" + pipeline = Pipeline() + git_ref = _get_git_ref() + + for p in PLATFORMS: + for bt in BUILD_TYPES: + for py in PYTHON_VERSIONS: + # TODO update slug logic in the submodule + slug = p.slug(bt.lower()) + f"-py{py.replace('.', '')}" + # We want to use only release builds as dependencies + dependency_slug = p.slug("release") + + build_id = os.environ.get("BUILDKITE_BUILD_ID", "local") + tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "build_id": build_id, "dependency_slug": dependency_slug, "ref": git_ref} + + # 1. Run tests + step = load_template(STEPS_DIR / "_test.yml", **tvars) + env = _env(p, "test", bt) + env.update(step.get("env") or {}) + env.update({"PYTHON_VERSION": py}) + step["env"] = env + pipeline.add_step(CommandStep.from_dict(step)) + + return pipeline + + +def main() -> None: + command = sys.argv[1] if len(sys.argv) > 1 else "generate" + + try: + pipeline = generate_pipeline() + except Exception as e: + print(f"[FAIL] Pipeline generation failed: {e}", file=sys.stderr) + sys.exit(1) + + if command == "generate": + print(pipeline.to_yaml()) + elif command == "check": + errors = validate_pipeline(pipeline) + if errors: + for e in errors: + print(f"[FAIL] {e}", file=sys.stderr) + sys.exit(1) + print(f"[OK] Pipeline valid: {len(pipeline.steps)} steps") + else: + print(f"Unknown command: {command}", file=sys.stderr) + print("Usage: pipeline.py [generate|check]", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.buildkite/requirements.txt b/.buildkite/requirements.txt new file mode 100644 index 000000000..971d0fe74 --- /dev/null +++ b/.buildkite/requirements.txt @@ -0,0 +1,2 @@ +buildkite-sdk==0.8.0 +-r tools/requirements.txt \ No newline at end of file diff --git a/.buildkite/steps/_test.yml b/.buildkite/steps/_test.yml new file mode 100644 index 000000000..207a3f85f --- /dev/null +++ b/.buildkite/steps/_test.yml @@ -0,0 +1,39 @@ +agents: + queue: "default-{queue}" + +label: "Test ({slug})" +key: "test-{slug}" + +commands: + - "echo \"+++ Start Services\"" + - "scripts/tests/setup/start-services.sh" + - "echo \"+++ Run Tests\"" + - "scripts/teamcity/20.test.sh" + - "echo \"+++ Run build\"" + - "scripts/teamcity/10.build.sh" + - "echo \"+++ Stop Services\"" + - "scripts/tests/setup/stop-services.sh" + +plugins: + - docker-compose#v5.12.1: + run: pypa + config: docker/docker-compose.yml + propagate-environment: true + propagate-uid-gid: true + + - bureau14/qdb-artifacts#artifact-dependency: + download: + project_id: "quasardb" + variant: "{dependency_slug}" + ref: "{ref}" + output-dir: "qdb" + extract: true + clean: true + files: + - "*-c-api.tar.zst!*" + - "*-server.tar.zst!*" + - "*-utils.tar.zst!*" + upload: + files: "dist/*" + promote: + step: "{slug}" diff --git a/.gitmodules b/.gitmodules index d97d4dac4..068c910f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = scripts/tests/setup url = https://github.com/bureau14/qdb-test-setup branch = master +[submodule ".buildkite/tools"] + path = .buildkite/tools + url = https://github.com/bureau14/qdb-cicd-tools.git diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 000000000..f10b54bb0 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,10 @@ +services: + pypa: + build: + context: . + dockerfile: Dockerfile + args: + - PYTHON_VERSION=${PYTHON_VERSION} + volumes: + - ../:/workdir + working_dir: /workdir From 7b4d1a533429cbb1b9a485823a3c6ae3da584600 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 10:41:14 +0200 Subject: [PATCH 02/58] enable running from compose --- docker/Dockerfile | 3 +++ docker/set-python-version.sh | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 78905214f..be406c175 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,3 +9,6 @@ ARG PYTHON_VERSION ADD set-python-version.sh /set-python-version.sh RUN bash /set-python-version.sh ${PYTHON_VERSION} \ && rm /set-python-version.sh + +#Packages required for running start-services inside the container +RUN yum install lsof -y diff --git a/docker/set-python-version.sh b/docker/set-python-version.sh index 6331e982b..fabdc1d13 100644 --- a/docker/set-python-version.sh +++ b/docker/set-python-version.sh @@ -45,5 +45,11 @@ PYTHON_BINDIR="/opt/python/${PARSED}/bin/" echo 'export PATH='$PYTHON_BINDIR':$PATH' | tee -a ~/.bashrc PATH=$PYTHON_BINDIR:$PATH +# Symlink to /usr/local/bin so it's available in non-interactive shells +ln -sf ${PYTHON_BINDIR}/python /usr/local/bin/python +ln -sf ${PYTHON_BINDIR}/python3 /usr/local/bin/python3 +ln -sf ${PYTHON_BINDIR}/pip /usr/local/bin/pip +ln -sf ${PYTHON_BINDIR}/pip3 /usr/local/bin/pip3 + # Install dependencies python3 --version && pip3 install --upgrade pip setuptools wheel From f52ee5cac9903f4f7013a580b96ce11fbe165f9d Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 10:41:21 +0200 Subject: [PATCH 03/58] ref override --- .buildkite/pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index ac1927797..5859f04db 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -165,7 +165,9 @@ def generate_pipeline() -> Pipeline: slug = p.slug(bt.lower()) + f"-py{py.replace('.', '')}" # We want to use only release builds as dependencies dependency_slug = p.slug("release") - + + # TODO: this is just for testing + git_ref = "refs/heads/artifact-dependency" build_id = os.environ.get("BUILDKITE_BUILD_ID", "local") tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "build_id": build_id, "dependency_slug": dependency_slug, "ref": git_ref} From 384935561075d1169ff0d9440e18df0a9f1c6223 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 10:44:44 +0200 Subject: [PATCH 04/58] change dep Co-authored-by: Copilot --- .buildkite/pipeline.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 5859f04db..615e238ff 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -36,16 +36,16 @@ # cxx_compiler="/usr/local/gcc13/bin/g++", # asm_compiler="/usr/local/bin/yasm", # ccache="/usr/local/bin/ccache", - docker_image="bureau14/builder:rhel7", + # docker_image="bureau14/builder:rhel7", # docker_volumes=("/var/lib/ccache:/var/lib/ccache",), ) _WIN = dict( - asm_compiler="C:\\yasm\\yasm-1.3.0-win64.exe", + # asm_compiler="C:\\yasm\\yasm-1.3.0-win64.exe", # ccache="C:\\ccache\\ccache.exe", ) _MACOS = dict( - c_compiler="/usr/local/clang21/bin/clang", - cxx_compiler="/usr/local/clang21/bin/clang++", + # c_compiler="/usr/local/clang21/bin/clang", + # cxx_compiler="/usr/local/clang21/bin/clang++", # ccache="/opt/local/bin/ccache", ) _OS_OVERLAY = {"linux": _LINUX, "windows": _WIN, "macos": _MACOS} @@ -81,18 +81,12 @@ # ), # "CCACHE_RESHARE": "true", "AWS_DEFAULT_REGION": "eu-west-1", - "QDB_ENCRYPT_TRAFFIC": "1", "PYTHON_EXECUTABLE": "/usr/bin/python3", "PYTHON_CMD": "python3", } STEP_ENV: dict[str, dict[str, str]] = { - # "build_docker": {}, - # "start_services": { - # }, - # "test": { - # }, - # "stop_services": {}, + "test": {"QDB_ENCRYPT_TRAFFIC": "1",}, } OS_ENV: dict[str, dict[str, str]] = { @@ -167,7 +161,7 @@ def generate_pipeline() -> Pipeline: dependency_slug = p.slug("release") # TODO: this is just for testing - git_ref = "refs/heads/artifact-dependency" + git_ref = "refs/heads/sc-18547/buildkite" build_id = os.environ.get("BUILDKITE_BUILD_ID", "local") tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "build_id": build_id, "dependency_slug": dependency_slug, "ref": git_ref} From 114c83be0f269fee7043666c3d0de357f7e09c09 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 11:02:53 +0200 Subject: [PATCH 05/58] submodule --- .buildkite/tools | 1 + 1 file changed, 1 insertion(+) create mode 160000 .buildkite/tools diff --git a/.buildkite/tools b/.buildkite/tools new file mode 160000 index 000000000..10a298957 --- /dev/null +++ b/.buildkite/tools @@ -0,0 +1 @@ +Subproject commit 10a29895745adb1e26094ed1f489d45923fecc96 From 1991b8d0d620d93d00e3ed866d34e8adc9388641 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 11:04:08 +0200 Subject: [PATCH 06/58] param name changed --- .buildkite/steps/_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/_test.yml b/.buildkite/steps/_test.yml index 207a3f85f..efe2ee245 100644 --- a/.buildkite/steps/_test.yml +++ b/.buildkite/steps/_test.yml @@ -25,7 +25,7 @@ plugins: download: project_id: "quasardb" variant: "{dependency_slug}" - ref: "{ref}" + git_ref: "{ref}" output-dir: "qdb" extract: true clean: true From 8b6220277d5383e242d825801a04ba14d90c46b0 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 11:12:31 +0200 Subject: [PATCH 07/58] fix upload/promote step Co-authored-by: Copilot --- .buildkite/steps/_test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.buildkite/steps/_test.yml b/.buildkite/steps/_test.yml index efe2ee245..4bca6e12a 100644 --- a/.buildkite/steps/_test.yml +++ b/.buildkite/steps/_test.yml @@ -9,10 +9,10 @@ commands: - "scripts/tests/setup/start-services.sh" - "echo \"+++ Run Tests\"" - "scripts/teamcity/20.test.sh" - - "echo \"+++ Run build\"" - - "scripts/teamcity/10.build.sh" - "echo \"+++ Stop Services\"" - "scripts/tests/setup/stop-services.sh" + - "echo \"+++ Run build\"" + - "scripts/teamcity/10.build.sh" plugins: - docker-compose#v5.12.1: @@ -35,5 +35,8 @@ plugins: - "*-utils.tar.zst!*" upload: files: "dist/*" + variant: "{slug}" + git_ref: "{ref}" promote: - step: "{slug}" + variant: "{slug}" + git_ref: "{ref}" From cb8c484ac69bd014d872d49c1045cdd9a1e0d29a Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 11:31:07 +0200 Subject: [PATCH 08/58] try windows Co-authored-by: Copilot --- .buildkite/pipeline.py | 46 +++++++++++++++++++++++++------------- .buildkite/steps/_test.yml | 2 -- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 615e238ff..a1a52642c 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -32,22 +32,12 @@ # Quasardb-specific toolchain overlays on top of shared infrastructure platforms. _LINUX = dict( - # c_compiler="/usr/local/gcc13/bin/gcc", - # cxx_compiler="/usr/local/gcc13/bin/g++", - # asm_compiler="/usr/local/bin/yasm", - # ccache="/usr/local/bin/ccache", - # docker_image="bureau14/builder:rhel7", - # docker_volumes=("/var/lib/ccache:/var/lib/ccache",), ) _WIN = dict( - # asm_compiler="C:\\yasm\\yasm-1.3.0-win64.exe", - # ccache="C:\\ccache\\ccache.exe", ) _MACOS = dict( - # c_compiler="/usr/local/clang21/bin/clang", - # cxx_compiler="/usr/local/clang21/bin/clang++", - # ccache="/opt/local/bin/ccache", ) + _OS_OVERLAY = {"linux": _LINUX, "windows": _WIN, "macos": _MACOS} PLATFORMS: list[Platform] = [ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) @@ -55,8 +45,7 @@ "linux-amd64-core2", # "linux-amd64-haswell", # "linux-aarch64", - # "windows-amd64-core2", - # "windows-amd64-haswell", + "windows-amd64-core2", # "macos-aarch64", ) ] @@ -147,6 +136,27 @@ def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str]) -> None: if key not in config: config[key] = vars[key] +def _apply_docker_compose( + step: dict, + config: dict[str, str] = {}, +) -> None: + DOCKER_COMPOSE_PLUGIN_VERSION="v5.12.1" + + + docker_plugin = { + f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { + "run": "pypa", + "config": "docker/docker-compose.yml", + "propagate-environment": True, + "propagate-uid-gid": True, + **config, + }, + } + existing = step.get("plugins", []) + step["plugins"] = [docker_plugin] + existing + + + def generate_pipeline() -> Pipeline: """Load templates, expand across platforms × build_types, overlay env and docker.""" pipeline = Pipeline() @@ -162,15 +172,19 @@ def generate_pipeline() -> Pipeline: # TODO: this is just for testing git_ref = "refs/heads/sc-18547/buildkite" - build_id = os.environ.get("BUILDKITE_BUILD_ID", "local") - tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "build_id": build_id, "dependency_slug": dependency_slug, "ref": git_ref} + # + tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "dependency_slug": dependency_slug, "ref": git_ref} - # 1. Run tests step = load_template(STEPS_DIR / "_test.yml", **tvars) env = _env(p, "test", bt) env.update(step.get("env") or {}) env.update({"PYTHON_VERSION": py}) + if p.os == "windows": + env["PYTHON_CMD"] = r"C:\Python3.14-64\python.exe" + env["PYTHON_EXECUTABLE"] = r"C:\Python3.14-64\python.exe" step["env"] = env + if p.os == "linux": + _apply_docker_compose(step) pipeline.add_step(CommandStep.from_dict(step)) return pipeline diff --git a/.buildkite/steps/_test.yml b/.buildkite/steps/_test.yml index 4bca6e12a..9d6fa52a8 100644 --- a/.buildkite/steps/_test.yml +++ b/.buildkite/steps/_test.yml @@ -16,8 +16,6 @@ commands: plugins: - docker-compose#v5.12.1: - run: pypa - config: docker/docker-compose.yml propagate-environment: true propagate-uid-gid: true From 517be400494c5ecfdb1e1945ea8c6756d0180b0d Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 16:32:30 +0200 Subject: [PATCH 09/58] set env vars Co-authored-by: Copilot --- .buildkite/pipeline.py | 52 +++++++++++++++++++++++++------------- .buildkite/steps/_test.yml | 10 +------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index a1a52642c..9abc502d9 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -43,7 +43,6 @@ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( "linux-amd64-core2", - # "linux-amd64-haswell", # "linux-aarch64", "windows-amd64-core2", # "macos-aarch64", @@ -63,12 +62,6 @@ # Environment variable layering: global → step → os → os+step → platform compilers. GLOBAL_ENV: dict[str, str] = { - # "CCACHE_COMPILERCHECK": "%compiler% -dumpmachine; %compiler% -dumpversion", - # "CCACHE_REMOTE_STORAGE": ( - # "http://ccache.cicd.intra.quasar.ai/" - # "|connect-timeout=1000|operation-timeout=30000|keep-alive=true" - # ), - # "CCACHE_RESHARE": "true", "AWS_DEFAULT_REGION": "eu-west-1", "PYTHON_EXECUTABLE": "/usr/bin/python3", "PYTHON_CMD": "python3", @@ -119,7 +112,7 @@ def _get_git_ref() -> str: return ref -def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str]) -> None: +def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str], plugin_steps: set[str] = None) -> None: """ Goes through list of plugins and fills in defaults for qdb-artifacts plugin if present. Doesn't overwrite existing keys, only fills in missing ones from provided dict. @@ -131,10 +124,34 @@ def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str]) -> None: if plugin_name.startswith("bureau14/qdb-artifacts#"): plugin_config = plugin_dict[plugin_name] - for config in plugin_config.values(): - for key in vars: - if key not in config: - config[key] = vars[key] + + if plugin_steps is None: + plugin_steps = set(plugin_config.keys()) + + for plugin_step, config in plugin_config.items(): + if plugin_step in plugin_steps: + for key in vars: + if key not in config: + config[key] = vars[key] + +def _get_agent_python_env(step: dict, platform: Platform, python_version: str) -> dict[str, str]: + env = step.get("env", {}) + + # TODO (igor) + # we can rely on referencing env variables instead of hardcoding paths per version and platform + # we need to update agents first to support this + python_version_path = python_version.replace('.', '') + if platform.os == "windows": + return { + "PYTHON_EXECUTABLE": f"C:\\Python{python_version_path}-64\\python.exe", + "PYTHON_CMD": f"C:\\Python{python_version_path}-64\\python.exe", + } + elif platform.os == "macos": + return { + "PYTHON_EXECUTABLE": f"/opt/local/bin/python{python_version}", + "PYTHON_CMD": f"/opt/local/bin/python{python_version}" + } + return {} def _apply_docker_compose( step: dict, @@ -171,20 +188,21 @@ def generate_pipeline() -> Pipeline: dependency_slug = p.slug("release") # TODO: this is just for testing - git_ref = "refs/heads/sc-18547/buildkite" + git_ref_dep = "refs/heads/sc-18547/buildkite" # - tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "dependency_slug": dependency_slug, "ref": git_ref} + tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "dependency_slug": dependency_slug, "ref": git_ref_dep} + artifact_upload_vars = {"variant": slug, "ref": git_ref} + plugin_steps = {"upload", "promote"} step = load_template(STEPS_DIR / "_test.yml", **tvars) env = _env(p, "test", bt) env.update(step.get("env") or {}) env.update({"PYTHON_VERSION": py}) - if p.os == "windows": - env["PYTHON_CMD"] = r"C:\Python3.14-64\python.exe" - env["PYTHON_EXECUTABLE"] = r"C:\Python3.14-64\python.exe" + env.update(_get_agent_python_env(step, p, py)) step["env"] = env if p.os == "linux": _apply_docker_compose(step) + _set_artifact_plugin_defaults(step, artifact_upload_vars, plugin_steps) pipeline.add_step(CommandStep.from_dict(step)) return pipeline diff --git a/.buildkite/steps/_test.yml b/.buildkite/steps/_test.yml index 9d6fa52a8..fe13d56d3 100644 --- a/.buildkite/steps/_test.yml +++ b/.buildkite/steps/_test.yml @@ -15,10 +15,6 @@ commands: - "scripts/teamcity/10.build.sh" plugins: - - docker-compose#v5.12.1: - propagate-environment: true - propagate-uid-gid: true - - bureau14/qdb-artifacts#artifact-dependency: download: project_id: "quasardb" @@ -33,8 +29,4 @@ plugins: - "*-utils.tar.zst!*" upload: files: "dist/*" - variant: "{slug}" - git_ref: "{ref}" - promote: - variant: "{slug}" - git_ref: "{ref}" + promote: {} \ No newline at end of file From 8383179a9e8b042b97ef61e465453dec54369c6d Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 16:40:57 +0200 Subject: [PATCH 10/58] we changed from ref to git ref... --- .buildkite/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 9abc502d9..a75e7601e 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -44,7 +44,7 @@ for p in select_platforms( "linux-amd64-core2", # "linux-aarch64", - "windows-amd64-core2", + # "windows-amd64-core2", # "macos-aarch64", ) ] @@ -191,7 +191,7 @@ def generate_pipeline() -> Pipeline: git_ref_dep = "refs/heads/sc-18547/buildkite" # tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "dependency_slug": dependency_slug, "ref": git_ref_dep} - artifact_upload_vars = {"variant": slug, "ref": git_ref} + artifact_upload_vars = {"variant": slug, "git-ref": git_ref} plugin_steps = {"upload", "promote"} step = load_template(STEPS_DIR / "_test.yml", **tvars) From 398dd0625f3791f4e68eac8e100106dcfcef202c Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 16:42:37 +0200 Subject: [PATCH 11/58] all thats required to make scripts work on windows? Co-authored-by: Copilot --- .buildkite/steps/_test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.buildkite/steps/_test.yml b/.buildkite/steps/_test.yml index fe13d56d3..4afdd50ac 100644 --- a/.buildkite/steps/_test.yml +++ b/.buildkite/steps/_test.yml @@ -5,14 +5,14 @@ label: "Test ({slug})" key: "test-{slug}" commands: - - "echo \"+++ Start Services\"" - - "scripts/tests/setup/start-services.sh" - - "echo \"+++ Run Tests\"" - - "scripts/teamcity/20.test.sh" - - "echo \"+++ Stop Services\"" - - "scripts/tests/setup/stop-services.sh" - - "echo \"+++ Run build\"" - - "scripts/teamcity/10.build.sh" + - echo \"+++ Start Services\" + - bash scripts/tests/setup/start-services.sh + - echo \"+++ Run Tests\" + - bash scripts/teamcity/20.test.sh + - echo \"+++ Stop Services\" + - bash scripts/tests/setup/stop-services.sh + - echo \"+++ Run build\" + - bash scripts/teamcity/10.build.sh plugins: - bureau14/qdb-artifacts#artifact-dependency: From e3f97470019e139f35bf15637543b1e040d62ad5 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 16:57:08 +0200 Subject: [PATCH 12/58] cant run if disabled :) --- .buildkite/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index a75e7601e..12d04406d 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -43,8 +43,8 @@ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( "linux-amd64-core2", - # "linux-aarch64", - # "windows-amd64-core2", + "linux-aarch64", + "windows-amd64-core2", # "macos-aarch64", ) ] From 5a4f33db8a513b0c808060eb2d1e1ae7e5a1cd25 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 23 Apr 2026 17:13:47 +0200 Subject: [PATCH 13/58] python path, aarch64 arch Co-authored-by: Copilot --- .buildkite/pipeline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 12d04406d..7c45f8a65 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -75,7 +75,7 @@ # "linux": {"CCACHE_DIR": "/var/lib/ccache"}, # "freebsd": {}, # "macos": {}, - # "windows": {"CCACHE_COMPILERCHECK": ""}, + # "windows": {}, } OS_STEP_ENV: dict[str, dict[str, str]] = { @@ -84,6 +84,7 @@ CPU_ENV: dict[str, dict[str, str]] = { # "core2": {"QDB_CPU_ARCHITECTURE_CORE2": "ON"}, + "aarch64": {"ARCH": "aarch64"}, } @@ -140,11 +141,10 @@ def _get_agent_python_env(step: dict, platform: Platform, python_version: str) - # TODO (igor) # we can rely on referencing env variables instead of hardcoding paths per version and platform # we need to update agents first to support this - python_version_path = python_version.replace('.', '') if platform.os == "windows": return { - "PYTHON_EXECUTABLE": f"C:\\Python{python_version_path}-64\\python.exe", - "PYTHON_CMD": f"C:\\Python{python_version_path}-64\\python.exe", + "PYTHON_EXECUTABLE": f"C:\\Python{python_version}-64\\python.exe", + "PYTHON_CMD": f"C:\\Python{python_version}-64\\python.exe", } elif platform.os == "macos": return { From 0e08aa2a91403b8939c2b824a89083e60becded4 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 08:49:58 +0200 Subject: [PATCH 14/58] pass arch to container Co-authored-by: Copilot --- .buildkite/pipeline.py | 23 +++++++++++++---------- docker/docker-compose.yml | 1 + 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 7c45f8a65..4e6fcfa63 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -83,7 +83,6 @@ } CPU_ENV: dict[str, dict[str, str]] = { - # "core2": {"QDB_CPU_ARCHITECTURE_CORE2": "ON"}, "aarch64": {"ARCH": "aarch64"}, } @@ -136,11 +135,12 @@ def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str], plugin_steps config[key] = vars[key] def _get_agent_python_env(step: dict, platform: Platform, python_version: str) -> dict[str, str]: - env = step.get("env", {}) - - # TODO (igor) - # we can rely on referencing env variables instead of hardcoding paths per version and platform - # we need to update agents first to support this + """ + Returns environment variables to set for Python executable on the agent, based on platform and python version. + Applies to Windows and macOS where we have multiple Python versions installed in different locations. + """ + # XXX (igor) + # we can rely on referencing env variables instead of hardcoded paths but we need to update agents first to support this if platform.os == "windows": return { "PYTHON_EXECUTABLE": f"C:\\Python{python_version}-64\\python.exe", @@ -162,9 +162,6 @@ def _apply_docker_compose( docker_plugin = { f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { - "run": "pypa", - "config": "docker/docker-compose.yml", - "propagate-environment": True, "propagate-uid-gid": True, **config, }, @@ -194,6 +191,12 @@ def generate_pipeline() -> Pipeline: artifact_upload_vars = {"variant": slug, "git-ref": git_ref} plugin_steps = {"upload", "promote"} + compose_config = { + "run": "pypa", + "config": "docker/docker-compose.yml", + "propagate-environment": True, + } + step = load_template(STEPS_DIR / "_test.yml", **tvars) env = _env(p, "test", bt) env.update(step.get("env") or {}) @@ -201,7 +204,7 @@ def generate_pipeline() -> Pipeline: env.update(_get_agent_python_env(step, p, py)) step["env"] = env if p.os == "linux": - _apply_docker_compose(step) + _apply_docker_compose(step, compose_config) _set_artifact_plugin_defaults(step, artifact_upload_vars, plugin_steps) pipeline.add_step(CommandStep.from_dict(step)) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f10b54bb0..7dc4b2998 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,6 +5,7 @@ services: dockerfile: Dockerfile args: - PYTHON_VERSION=${PYTHON_VERSION} + - ARCH=${ARCH-x86_64} volumes: - ../:/workdir working_dir: /workdir From 828c8decb62646136037cf5baf95b91fe8fa6c30 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 09:22:48 +0200 Subject: [PATCH 15/58] cleanup Co-authored-by: Copilot --- .buildkite/pipeline.py | 45 +++++++++++----------- .buildkite/steps/{_test.yml => _build.yml} | 12 +++--- 2 files changed, 30 insertions(+), 27 deletions(-) rename .buildkite/steps/{_test.yml => _build.yml} (74%) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 4e6fcfa63..e7620c35a 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -45,11 +45,11 @@ "linux-amd64-core2", "linux-aarch64", "windows-amd64-core2", - # "macos-aarch64", + "macos-aarch64", ) ] -BUILD_TYPES = ["Release"] # Temp +BUILD_TYPES = ["Release", "Debug"] PYTHON_VERSIONS = [ # "3.9", @@ -72,14 +72,14 @@ } OS_ENV: dict[str, dict[str, str]] = { - # "linux": {"CCACHE_DIR": "/var/lib/ccache"}, - # "freebsd": {}, - # "macos": {}, - # "windows": {}, + "linux": {}, + "freebsd": {}, + "macos": {}, + "windows": {}, } OS_STEP_ENV: dict[str, dict[str, str]] = { - # "linux/build": {"QDB_ENABLE_API_DOCS": "ON"}, + } CPU_ENV: dict[str, dict[str, str]] = { @@ -112,7 +112,7 @@ def _get_git_ref() -> str: return ref -def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str], plugin_steps: set[str] = None) -> None: +def _set_artifact_plugin_defaults(step: dict, vars_per_step: dict[str, dict[str, str]]) -> None: """ Goes through list of plugins and fills in defaults for qdb-artifacts plugin if present. Doesn't overwrite existing keys, only fills in missing ones from provided dict. @@ -124,15 +124,11 @@ def _set_artifact_plugin_defaults(step: dict, vars: dict[str, str], plugin_steps if plugin_name.startswith("bureau14/qdb-artifacts#"): plugin_config = plugin_dict[plugin_name] - - if plugin_steps is None: - plugin_steps = set(plugin_config.keys()) - for plugin_step, config in plugin_config.items(): - if plugin_step in plugin_steps: - for key in vars: + if plugin_step in vars_per_step.keys(): + for key in vars_per_step[plugin_step]: if key not in config: - config[key] = vars[key] + config[key] = vars_per_step[plugin_step][key] def _get_agent_python_env(step: dict, platform: Platform, python_version: str) -> dict[str, str]: """ @@ -181,15 +177,20 @@ def generate_pipeline() -> Pipeline: for py in PYTHON_VERSIONS: # TODO update slug logic in the submodule slug = p.slug(bt.lower()) + f"-py{py.replace('.', '')}" - # We want to use only release builds as dependencies + + # We want to use Release QuasarDB binaries for building debug Python API dependency_slug = p.slug("release") - + # TODO: this is just for testing git_ref_dep = "refs/heads/sc-18547/buildkite" # - tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "dependency_slug": dependency_slug, "ref": git_ref_dep} - artifact_upload_vars = {"variant": slug, "git-ref": git_ref} - plugin_steps = {"upload", "promote"} + tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "build_type": bt, "python_version": py} + + artifact_vars_per_step = { + "upload": {"variant": slug, "git-ref": git_ref}, + "promote": {"variant": slug, "git-ref": git_ref}, + "download": {"variant": dependency_slug, "git-ref": git_ref_dep}, + } compose_config = { "run": "pypa", @@ -197,7 +198,7 @@ def generate_pipeline() -> Pipeline: "propagate-environment": True, } - step = load_template(STEPS_DIR / "_test.yml", **tvars) + step = load_template(STEPS_DIR / "_build.yml", **tvars) env = _env(p, "test", bt) env.update(step.get("env") or {}) env.update({"PYTHON_VERSION": py}) @@ -205,7 +206,7 @@ def generate_pipeline() -> Pipeline: step["env"] = env if p.os == "linux": _apply_docker_compose(step, compose_config) - _set_artifact_plugin_defaults(step, artifact_upload_vars, plugin_steps) + _set_artifact_plugin_defaults(step, artifact_vars_per_step) pipeline.add_step(CommandStep.from_dict(step)) return pipeline diff --git a/.buildkite/steps/_test.yml b/.buildkite/steps/_build.yml similarity index 74% rename from .buildkite/steps/_test.yml rename to .buildkite/steps/_build.yml index 4afdd50ac..ba0e7ce71 100644 --- a/.buildkite/steps/_test.yml +++ b/.buildkite/steps/_build.yml @@ -1,8 +1,8 @@ agents: queue: "default-{queue}" -label: "Test ({slug})" -key: "test-{slug}" +label: "{build_type} Python {python_version} ({slug})" +key: "build-{slug}" commands: - echo \"+++ Start Services\" @@ -18,8 +18,6 @@ plugins: - bureau14/qdb-artifacts#artifact-dependency: download: project_id: "quasardb" - variant: "{dependency_slug}" - git_ref: "{ref}" output-dir: "qdb" extract: true clean: true @@ -28,5 +26,9 @@ plugins: - "*-server.tar.zst!*" - "*-utils.tar.zst!*" upload: - files: "dist/*" + files: + - "dist/quasardb*.whl" + - "dist/quasardb*.tar.gz" + - "dist/doc.tar.gz" + - "dist/quasardb*.egg" promote: {} \ No newline at end of file From 9f704ae93b4916d0ad8551064d41251e9ec013b8 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 09:29:51 +0200 Subject: [PATCH 16/58] update naming Co-authored-by: Copilot --- .buildkite/pipeline.py | 12 ++++++------ .buildkite/steps/_build.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index e7620c35a..b7527e009 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -52,11 +52,11 @@ BUILD_TYPES = ["Release", "Debug"] PYTHON_VERSIONS = [ - # "3.9", - # "3.10", - # "3.11", - # "3.12", - # "3.13", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", "3.14", ] @@ -184,7 +184,7 @@ def generate_pipeline() -> Pipeline: # TODO: this is just for testing git_ref_dep = "refs/heads/sc-18547/buildkite" # - tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "build_type": bt, "python_version": py} + tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "name": slug.replace("-", " ").title()} artifact_vars_per_step = { "upload": {"variant": slug, "git-ref": git_ref}, diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index ba0e7ce71..119c594be 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -1,7 +1,7 @@ agents: queue: "default-{queue}" -label: "{build_type} Python {python_version} ({slug})" +label: "{name} ({slug})" key: "build-{slug}" commands: From 139760a43d9dacc5ac38b8e0b60a84b0e45bd2f4 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 10:29:00 +0200 Subject: [PATCH 17/58] add groups Co-authored-by: Copilot --- .buildkite/pipeline.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index b7527e009..fcc4e614c 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -16,7 +16,7 @@ from pathlib import Path import os -from buildkite_sdk import CommandStep, Pipeline +from buildkite_sdk import Pipeline, GroupStep sys.path.insert(0, str(Path(__file__).parent / "tools")) from qdb_pipeline import ( @@ -171,6 +171,7 @@ def generate_pipeline() -> Pipeline: """Load templates, expand across platforms × build_types, overlay env and docker.""" pipeline = Pipeline() git_ref = _get_git_ref() + key_per_group = {} for p in PLATFORMS: for bt in BUILD_TYPES: @@ -207,7 +208,17 @@ def generate_pipeline() -> Pipeline: if p.os == "linux": _apply_docker_compose(step, compose_config) _set_artifact_plugin_defaults(step, artifact_vars_per_step) - pipeline.add_step(CommandStep.from_dict(step)) + + # group builds by platform and build type + # we will see if it improves readability of the pipeline view in the UI + group_name = p.slug(bt.lower()).replace("-", " ").title() + if group_name not in key_per_group: + key_per_group[group_name] = [] + key_per_group[group_name].append(step) + + for group, steps in key_per_group.items(): + group_step = GroupStep(group=group, steps=steps) + pipeline.add_step(group_step) return pipeline From 80ed4c23501302232d67159cea808e08cfeeb7b6 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 10:54:33 +0200 Subject: [PATCH 18/58] retry on spot fleet termination --- .buildkite/steps/_build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index 119c594be..8f081a1f2 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -1,6 +1,11 @@ agents: queue: "default-{queue}" +retry: + automatic: + - signal_reason: agent_stop + limit: 3 + label: "{name} ({slug})" key: "build-{slug}" From d09e8e4ba5c7e7cb6967bde9c267fd7d853f2b4d Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 11:26:58 +0200 Subject: [PATCH 19/58] upload step supports only one expression --- .buildkite/steps/_build.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index 8f081a1f2..f01d96b1a 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -31,9 +31,5 @@ plugins: - "*-server.tar.zst!*" - "*-utils.tar.zst!*" upload: - files: - - "dist/quasardb*.whl" - - "dist/quasardb*.tar.gz" - - "dist/doc.tar.gz" - - "dist/quasardb*.egg" + files: "dist/*" promote: {} \ No newline at end of file From 2f0f90fe8cc026ef14532ea9cae170d43352c2db Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 11:33:21 +0200 Subject: [PATCH 20/58] change retry logic --- .buildkite/steps/_build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index f01d96b1a..dc43cfd4d 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -3,7 +3,6 @@ agents: retry: automatic: - - signal_reason: agent_stop limit: 3 label: "{name} ({slug})" From ee2bdca278ab5c0a9199e03369d114f97a1a3a6e Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 12:21:33 +0200 Subject: [PATCH 21/58] format Co-authored-by: Copilot --- .buildkite/pipeline.py | 74 +++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index fcc4e614c..b070868bd 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -31,12 +31,9 @@ STEPS_DIR = Path(__file__).parent / "steps" # Quasardb-specific toolchain overlays on top of shared infrastructure platforms. -_LINUX = dict( -) -_WIN = dict( -) -_MACOS = dict( -) +_LINUX = dict() +_WIN = dict() +_MACOS = dict() _OS_OVERLAY = {"linux": _LINUX, "windows": _WIN, "macos": _MACOS} PLATFORMS: list[Platform] = [ @@ -63,24 +60,26 @@ # Environment variable layering: global → step → os → os+step → platform compilers. GLOBAL_ENV: dict[str, str] = { "AWS_DEFAULT_REGION": "eu-west-1", - "PYTHON_EXECUTABLE": "/usr/bin/python3", - "PYTHON_CMD": "python3", + "JUNIT_XML_FILE": "build/test/pytest.xml", + "QDB_ENCRYPT_TRAFFIC": "1", } -STEP_ENV: dict[str, dict[str, str]] = { - "test": {"QDB_ENCRYPT_TRAFFIC": "1",}, -} +STEP_ENV: dict[str, dict[str, str]] = {} OS_ENV: dict[str, dict[str, str]] = { - "linux": {}, - "freebsd": {}, + "linux": { + "PYTHON_EXECUTABLE": "/usr/bin/python3", + "PYTHON_CMD": "python3", + }, + "freebsd": { + "PYTHON_EXECUTABLE": "/usr/bin/python3", + "PYTHON_CMD": "python3", + }, "macos": {}, "windows": {}, } -OS_STEP_ENV: dict[str, dict[str, str]] = { - -} +OS_STEP_ENV: dict[str, dict[str, str]] = {} CPU_ENV: dict[str, dict[str, str]] = { "aarch64": {"ARCH": "aarch64"}, @@ -99,6 +98,7 @@ def _env(p: Platform, step_name: str, build_type: str) -> dict[str, str]: platform=p, ) + def _get_git_ref() -> str: branch = os.environ.get("BUILDKITE_BRANCH") tag = os.environ.get("BUILDKITE_TAG") @@ -112,7 +112,9 @@ def _get_git_ref() -> str: return ref -def _set_artifact_plugin_defaults(step: dict, vars_per_step: dict[str, dict[str, str]]) -> None: +def _set_artifact_plugin_defaults( + step: dict, vars_per_step: dict[str, dict[str, str]] +) -> None: """ Goes through list of plugins and fills in defaults for qdb-artifacts plugin if present. Doesn't overwrite existing keys, only fills in missing ones from provided dict. @@ -130,7 +132,10 @@ def _set_artifact_plugin_defaults(step: dict, vars_per_step: dict[str, dict[str, if key not in config: config[key] = vars_per_step[plugin_step][key] -def _get_agent_python_env(step: dict, platform: Platform, python_version: str) -> dict[str, str]: + +def _get_agent_python_env( + platform: Platform, python_version: str +) -> dict[str, str]: """ Returns environment variables to set for Python executable on the agent, based on platform and python version. Applies to Windows and macOS where we have multiple Python versions installed in different locations. @@ -145,16 +150,16 @@ def _get_agent_python_env(step: dict, platform: Platform, python_version: str) - elif platform.os == "macos": return { "PYTHON_EXECUTABLE": f"/opt/local/bin/python{python_version}", - "PYTHON_CMD": f"/opt/local/bin/python{python_version}" + "PYTHON_CMD": f"/opt/local/bin/python{python_version}", } return {} + def _apply_docker_compose( step: dict, config: dict[str, str] = {}, ) -> None: - DOCKER_COMPOSE_PLUGIN_VERSION="v5.12.1" - + DOCKER_COMPOSE_PLUGIN_VERSION = "v5.12.1" docker_plugin = { f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { @@ -166,12 +171,11 @@ def _apply_docker_compose( step["plugins"] = [docker_plugin] + existing - def generate_pipeline() -> Pipeline: """Load templates, expand across platforms × build_types, overlay env and docker.""" pipeline = Pipeline() git_ref = _get_git_ref() - key_per_group = {} + group_steps = {} for p in PLATFORMS: for bt in BUILD_TYPES: @@ -185,13 +189,17 @@ def generate_pipeline() -> Pipeline: # TODO: this is just for testing git_ref_dep = "refs/heads/sc-18547/buildkite" # - tvars = {"slug": slug, "queue": f"{p.queue_os}-{p.arch}", "name": slug.replace("-", " ").title()} + tvars = { + "slug": slug, + "queue": f"{p.queue_os}-{p.arch}", + "name": slug.replace("-", " ").title(), + } artifact_vars_per_step = { "upload": {"variant": slug, "git-ref": git_ref}, "promote": {"variant": slug, "git-ref": git_ref}, "download": {"variant": dependency_slug, "git-ref": git_ref_dep}, - } + } compose_config = { "run": "pypa", @@ -203,20 +211,20 @@ def generate_pipeline() -> Pipeline: env = _env(p, "test", bt) env.update(step.get("env") or {}) env.update({"PYTHON_VERSION": py}) - env.update(_get_agent_python_env(step, p, py)) + env.update(_get_agent_python_env(p, py)) step["env"] = env if p.os == "linux": _apply_docker_compose(step, compose_config) _set_artifact_plugin_defaults(step, artifact_vars_per_step) - - # group builds by platform and build type - # we will see if it improves readability of the pipeline view in the UI + + # add step to group group_name = p.slug(bt.lower()).replace("-", " ").title() - if group_name not in key_per_group: - key_per_group[group_name] = [] - key_per_group[group_name].append(step) + if group_name not in group_steps: + group_steps[group_name] = [] + group_steps[group_name].append(step) - for group, steps in key_per_group.items(): + # create groups and add to pipeline + for group, steps in group_steps.items(): group_step = GroupStep(group=group, steps=steps) pipeline.add_step(group_step) From de7770384f82e0787e21845dc10820c9ea12341a Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 12:26:13 +0200 Subject: [PATCH 22/58] try to export PATH instead of using symlinks Co-authored-by: Copilot --- docker/set-python-version.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/set-python-version.sh b/docker/set-python-version.sh index fabdc1d13..214c19201 100644 --- a/docker/set-python-version.sh +++ b/docker/set-python-version.sh @@ -45,11 +45,7 @@ PYTHON_BINDIR="/opt/python/${PARSED}/bin/" echo 'export PATH='$PYTHON_BINDIR':$PATH' | tee -a ~/.bashrc PATH=$PYTHON_BINDIR:$PATH -# Symlink to /usr/local/bin so it's available in non-interactive shells -ln -sf ${PYTHON_BINDIR}/python /usr/local/bin/python -ln -sf ${PYTHON_BINDIR}/python3 /usr/local/bin/python3 -ln -sf ${PYTHON_BINDIR}/pip /usr/local/bin/pip -ln -sf ${PYTHON_BINDIR}/pip3 /usr/local/bin/pip3 +export PATH # Install dependencies python3 --version && pip3 install --upgrade pip setuptools wheel From 9f0e00a1cdffa13a1baace16ee40eb9c8f766215 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 12:32:50 +0200 Subject: [PATCH 23/58] revert last commit --- docker/set-python-version.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/set-python-version.sh b/docker/set-python-version.sh index 214c19201..0cfa9b92b 100644 --- a/docker/set-python-version.sh +++ b/docker/set-python-version.sh @@ -45,7 +45,12 @@ PYTHON_BINDIR="/opt/python/${PARSED}/bin/" echo 'export PATH='$PYTHON_BINDIR':$PATH' | tee -a ~/.bashrc PATH=$PYTHON_BINDIR:$PATH -export PATH +# Buildkite runs commands in non-interactive shell - doesn't use ~/.bashrc +# We use symlinks to make desired python version available in PATH for both interactive and non-interactive shells +ln -sf ${PYTHON_BINDIR}/python /usr/local/bin/python +ln -sf ${PYTHON_BINDIR}/python3 /usr/local/bin/python3 +ln -sf ${PYTHON_BINDIR}/pip /usr/local/bin/pip +ln -sf ${PYTHON_BINDIR}/pip3 /usr/local/bin/pip3 # Install dependencies python3 --version && pip3 install --upgrade pip setuptools wheel From 867a744e8569d31d895994fefce716b2294136da Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 14:00:46 +0200 Subject: [PATCH 24/58] Add conditional doc build Co-authored-by: Copilot --- .buildkite/pipeline.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index b070868bd..1691529d9 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -133,9 +133,7 @@ def _set_artifact_plugin_defaults( config[key] = vars_per_step[plugin_step][key] -def _get_agent_python_env( - platform: Platform, python_version: str -) -> dict[str, str]: +def _get_agent_python_env(platform: Platform, python_version: str) -> dict[str, str]: """ Returns environment variables to set for Python executable on the agent, based on platform and python version. Applies to Windows and macOS where we have multiple Python versions installed in different locations. @@ -171,6 +169,19 @@ def _apply_docker_compose( step["plugins"] = [docker_plugin] + existing +def _apply_doc_command(step: dict, platform: Platform) -> None: + """ + Adds a command to the step to generate documentation using pdoc to linux-amd64-core2 platform builds. + """ + if platform.os == "linux" and platform.arch == "amd64" and platform.cpu == "core2": + doc_commands = [ + 'echo "+++ Build documentation"', + "bash scripts/teamcity/30.doc.sh", + ] + existing_commands = step.get("commands", []) + existing_commands += doc_commands + + def generate_pipeline() -> Pipeline: """Load templates, expand across platforms × build_types, overlay env and docker.""" pipeline = Pipeline() @@ -216,6 +227,7 @@ def generate_pipeline() -> Pipeline: if p.os == "linux": _apply_docker_compose(step, compose_config) _set_artifact_plugin_defaults(step, artifact_vars_per_step) + _apply_doc_command(step, p) # add step to group group_name = p.slug(bt.lower()).replace("-", " ").title() From 76f7f925458fc61669d96d61a55ca4f4d5f06fbf Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 14:59:41 +0200 Subject: [PATCH 25/58] fix doc build --- scripts/teamcity/30.doc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/teamcity/30.doc.sh b/scripts/teamcity/30.doc.sh index e4929e1c7..4fb4510c9 100755 --- a/scripts/teamcity/30.doc.sh +++ b/scripts/teamcity/30.doc.sh @@ -18,7 +18,7 @@ fi ${VENV_PYTHON} -m pip install -r dev-requirements.txt ${VENV_PYTHON} -m pip install --no-deps --force-reinstall dist/quasardb-*manylinux*.whl -${VENV_PYTHON} -m pip install --upgrade pydoc3 +${VENV_PYTHON} -m pip install --upgrade pdoc3==0.11.5 # To avoid conflicts with `quasardb` directory and `import quasardb` @@ -31,4 +31,4 @@ ${VENV_PYTHON} docgen.py popd -tar -czvf dist/doc.tar.gz -C doc/build . +tar -czvf dist/doc.tar.gz -C doc/build . \ No newline at end of file From f5a82464f73d37465632b103b071950ea35913c9 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 15:00:07 +0200 Subject: [PATCH 26/58] don't invoke through bash --- .buildkite/pipeline.py | 2 +- .buildkite/steps/_build.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 1691529d9..0d1497e5f 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -176,7 +176,7 @@ def _apply_doc_command(step: dict, platform: Platform) -> None: if platform.os == "linux" and platform.arch == "amd64" and platform.cpu == "core2": doc_commands = [ 'echo "+++ Build documentation"', - "bash scripts/teamcity/30.doc.sh", + "scripts/teamcity/30.doc.sh", ] existing_commands = step.get("commands", []) existing_commands += doc_commands diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index dc43cfd4d..6b3c80edc 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -10,13 +10,13 @@ key: "build-{slug}" commands: - echo \"+++ Start Services\" - - bash scripts/tests/setup/start-services.sh + - scripts/tests/setup/start-services.sh - echo \"+++ Run Tests\" - - bash scripts/teamcity/20.test.sh + - scripts/teamcity/20.test.sh - echo \"+++ Stop Services\" - - bash scripts/tests/setup/stop-services.sh + - scripts/tests/setup/stop-services.sh - echo \"+++ Run build\" - - bash scripts/teamcity/10.build.sh + - scripts/teamcity/10.build.sh plugins: - bureau14/qdb-artifacts#artifact-dependency: From 9052c28d33616d874a02c8050b293ee9a5674341 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 24 Apr 2026 15:30:15 +0200 Subject: [PATCH 27/58] revert Co-authored-by: Copilot --- .buildkite/pipeline.py | 2 +- .buildkite/steps/_build.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 0d1497e5f..1691529d9 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -176,7 +176,7 @@ def _apply_doc_command(step: dict, platform: Platform) -> None: if platform.os == "linux" and platform.arch == "amd64" and platform.cpu == "core2": doc_commands = [ 'echo "+++ Build documentation"', - "scripts/teamcity/30.doc.sh", + "bash scripts/teamcity/30.doc.sh", ] existing_commands = step.get("commands", []) existing_commands += doc_commands diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index 6b3c80edc..dc43cfd4d 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -10,13 +10,13 @@ key: "build-{slug}" commands: - echo \"+++ Start Services\" - - scripts/tests/setup/start-services.sh + - bash scripts/tests/setup/start-services.sh - echo \"+++ Run Tests\" - - scripts/teamcity/20.test.sh + - bash scripts/teamcity/20.test.sh - echo \"+++ Stop Services\" - - scripts/tests/setup/stop-services.sh + - bash scripts/tests/setup/stop-services.sh - echo \"+++ Run build\" - - scripts/teamcity/10.build.sh + - bash scripts/teamcity/10.build.sh plugins: - bureau14/qdb-artifacts#artifact-dependency: From c803b59194ac4c09ae24179931e1dc1f0f4dbd64 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 08:44:29 +0200 Subject: [PATCH 28/58] match --- scripts/teamcity/30.doc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/teamcity/30.doc.sh b/scripts/teamcity/30.doc.sh index 4fb4510c9..526de99ba 100755 --- a/scripts/teamcity/30.doc.sh +++ b/scripts/teamcity/30.doc.sh @@ -31,4 +31,4 @@ ${VENV_PYTHON} docgen.py popd -tar -czvf dist/doc.tar.gz -C doc/build . \ No newline at end of file +tar -czvf dist/doc.tar.gz -C doc/build . From 2a8f102a79e475dd6ef0d29c91d2d42ce84e917d Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 09:05:00 +0200 Subject: [PATCH 29/58] multiple artifact paths Co-authored-by: Copilot --- .buildkite/steps/_build.yml | 6 ++++-- scripts/teamcity/10.build.sh | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index dc43cfd4d..5a82e46c9 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -30,5 +30,7 @@ plugins: - "*-server.tar.zst!*" - "*-utils.tar.zst!*" upload: - files: "dist/*" - promote: {} \ No newline at end of file + files: + - "dist/quasardb-*manylinux*.whl" + - "dist/doc.tar.gz" + promote: {} diff --git a/scripts/teamcity/10.build.sh b/scripts/teamcity/10.build.sh index 100033ad1..38663e4e2 100755 --- a/scripts/teamcity/10.build.sh +++ b/scripts/teamcity/10.build.sh @@ -32,6 +32,7 @@ function relabel_wheel { else # ${AUDITWHEEL_PLAT} is defined in manylinux base docker image ${VENV_PYTHON} -m auditwheel repair "$wheel" --plat "$AUDITWHEEL_PLAT" -w dist/ + rm "$wheel" fi } From ac1c819303c7ae7ad004fea02f7e83c8cb920740 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 09:27:43 +0200 Subject: [PATCH 30/58] check plugin --- .buildkite/steps/_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index 5a82e46c9..f7d41d4a5 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -19,7 +19,7 @@ commands: - bash scripts/teamcity/10.build.sh plugins: - - bureau14/qdb-artifacts#artifact-dependency: + - bureau14/qdb-artifacts#more-selective-uploads: download: project_id: "quasardb" output-dir: "qdb" From 851da3d23327d17e8a5e2adb2b68f6198d6cd6e4 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 09:42:36 +0200 Subject: [PATCH 31/58] change upload --- .buildkite/steps/_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index f7d41d4a5..fc2114c41 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -31,6 +31,6 @@ plugins: - "*-utils.tar.zst!*" upload: files: - - "dist/quasardb-*manylinux*.whl" + - "dist/quasardb-*.whl" - "dist/doc.tar.gz" promote: {} From ae6755ade7f78cefae2a11b5bf3d9836f9cc81af Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 09:55:36 +0200 Subject: [PATCH 32/58] comments --- .buildkite/pipeline.py | 39 ++++++++++++++++++++----------------- .buildkite/steps/_build.yml | 2 +- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 1691529d9..228c1945a 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -111,6 +111,7 @@ def _get_git_ref() -> str: return ref +### Pending move to qdb-cicd-tools def _set_artifact_plugin_defaults( step: dict, vars_per_step: dict[str, dict[str, str]] @@ -133,6 +134,23 @@ def _set_artifact_plugin_defaults( config[key] = vars_per_step[plugin_step][key] +def _apply_docker_compose( + step: dict, + config: dict[str, str] = {}, +) -> None: + DOCKER_COMPOSE_PLUGIN_VERSION = "v5.12.1" + + docker_plugin = { + f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { + "propagate-uid-gid": True, + **config, + }, + } + existing = step.get("plugins", []) + step["plugins"] = [docker_plugin] + existing + +### + def _get_agent_python_env(platform: Platform, python_version: str) -> dict[str, str]: """ Returns environment variables to set for Python executable on the agent, based on platform and python version. @@ -153,22 +171,6 @@ def _get_agent_python_env(platform: Platform, python_version: str) -> dict[str, return {} -def _apply_docker_compose( - step: dict, - config: dict[str, str] = {}, -) -> None: - DOCKER_COMPOSE_PLUGIN_VERSION = "v5.12.1" - - docker_plugin = { - f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { - "propagate-uid-gid": True, - **config, - }, - } - existing = step.get("plugins", []) - step["plugins"] = [docker_plugin] + existing - - def _apply_doc_command(step: dict, platform: Platform) -> None: """ Adds a command to the step to generate documentation using pdoc to linux-amd64-core2 platform builds. @@ -193,11 +195,12 @@ def generate_pipeline() -> Pipeline: for py in PYTHON_VERSIONS: # TODO update slug logic in the submodule slug = p.slug(bt.lower()) + f"-py{py.replace('.', '')}" + # slug = p.slug(bt.lower(), py.replace('.', '')) - # We want to use Release QuasarDB binaries for building debug Python API + # We want to use Release QuasarDB binaries when building Python API (debug and release) dependency_slug = p.slug("release") - # TODO: this is just for testing + # TODO: this is just for testing, remove later git_ref_dep = "refs/heads/sc-18547/buildkite" # tvars = { diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index fc2114c41..a336c5bca 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -19,7 +19,7 @@ commands: - bash scripts/teamcity/10.build.sh plugins: - - bureau14/qdb-artifacts#more-selective-uploads: + - bureau14/qdb-artifacts#master: download: project_id: "quasardb" output-dir: "qdb" From 8cec77ad58a2c9ebb8cea936a515bc6e931c8b0a Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 10:26:21 +0200 Subject: [PATCH 33/58] debug this first Co-authored-by: Copilot --- .buildkite/pipeline.py | 17 +++++++++-------- scripts/teamcity/20.test.sh | 9 +++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 228c1945a..26d20464b 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -40,20 +40,20 @@ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( "linux-amd64-core2", - "linux-aarch64", - "windows-amd64-core2", - "macos-aarch64", + # "linux-aarch64", + # "windows-amd64-core2", + # "macos-aarch64", ) ] BUILD_TYPES = ["Release", "Debug"] PYTHON_VERSIONS = [ - "3.9", - "3.10", - "3.11", - "3.12", - "3.13", + # "3.9", + # "3.10", + # "3.11", + # "3.12", + # "3.13", "3.14", ] @@ -219,6 +219,7 @@ def generate_pipeline() -> Pipeline: "run": "pypa", "config": "docker/docker-compose.yml", "propagate-environment": True, + # "chown": True, } step = load_template(STEPS_DIR / "_build.yml", **tvars) diff --git a/scripts/teamcity/20.test.sh b/scripts/teamcity/20.test.sh index ea3512f8b..8edd21b30 100755 --- a/scripts/teamcity/20.test.sh +++ b/scripts/teamcity/20.test.sh @@ -10,6 +10,15 @@ set -u -x PYTHON="${PYTHON_CMD:-python3}" + +echo "Print User and Path info for debugging purposes" +echo "Current user: $(whoami)" +echo "Current user ID: $(id -u)" +echo "Current PATH: $PATH" +echo "Python executable: $(which ${PYTHON})" +echo "Python version: $(${PYTHON} --version)" + + ### # NOTE(leon): ### From c26f28df3eb9948206596f9bdc7f59db2c85f35c Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 10:30:21 +0200 Subject: [PATCH 34/58] disable propagate-uid-gid --- .buildkite/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 26d20464b..98615badc 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -142,7 +142,7 @@ def _apply_docker_compose( docker_plugin = { f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { - "propagate-uid-gid": True, + # "propagate-uid-gid": True, **config, }, } From 108dfc9e1c5aa52a89f8d39d1cb1d62a8c2a264d Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 10:46:00 +0200 Subject: [PATCH 35/58] fix user --- .buildkite/pipeline.py | 3 +-- docker/Dockerfile | 4 ++++ docker/set-python-version.sh | 7 ------- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 98615badc..e283bb6b0 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -142,7 +142,7 @@ def _apply_docker_compose( docker_plugin = { f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { - # "propagate-uid-gid": True, + "propagate-uid-gid": True, **config, }, } @@ -219,7 +219,6 @@ def generate_pipeline() -> Pipeline: "run": "pypa", "config": "docker/docker-compose.yml", "propagate-environment": True, - # "chown": True, } step = load_template(STEPS_DIR / "_build.yml", **tvars) diff --git a/docker/Dockerfile b/docker/Dockerfile index be406c175..7a80de73f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,6 +6,10 @@ FROM quay.io/pypa/$PLATFORM:${TAG} ARG PYTHON_VERSION +# We need to ensure that the builder user has the same UID and GID as the host buildkite agent to use the propagate-uid-gid feature of the docker plugin. +RUN groupadd -g 929 builder && useradd -m -s /bin/bash -u 929 -g 929 builder +USER builder + ADD set-python-version.sh /set-python-version.sh RUN bash /set-python-version.sh ${PYTHON_VERSION} \ && rm /set-python-version.sh diff --git a/docker/set-python-version.sh b/docker/set-python-version.sh index 0cfa9b92b..6331e982b 100644 --- a/docker/set-python-version.sh +++ b/docker/set-python-version.sh @@ -45,12 +45,5 @@ PYTHON_BINDIR="/opt/python/${PARSED}/bin/" echo 'export PATH='$PYTHON_BINDIR':$PATH' | tee -a ~/.bashrc PATH=$PYTHON_BINDIR:$PATH -# Buildkite runs commands in non-interactive shell - doesn't use ~/.bashrc -# We use symlinks to make desired python version available in PATH for both interactive and non-interactive shells -ln -sf ${PYTHON_BINDIR}/python /usr/local/bin/python -ln -sf ${PYTHON_BINDIR}/python3 /usr/local/bin/python3 -ln -sf ${PYTHON_BINDIR}/pip /usr/local/bin/pip -ln -sf ${PYTHON_BINDIR}/pip3 /usr/local/bin/pip3 - # Install dependencies python3 --version && pip3 install --upgrade pip setuptools wheel From 3377eaa46bebe2ba8448e061d408d160ea4626b7 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 11:15:39 +0200 Subject: [PATCH 36/58] will it work now? Co-authored-by: Copilot --- docker/Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7a80de73f..0ecb46416 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,13 +6,14 @@ FROM quay.io/pypa/$PLATFORM:${TAG} ARG PYTHON_VERSION +# Install packages required for running start-services inside the container +RUN yum install lsof -y + # We need to ensure that the builder user has the same UID and GID as the host buildkite agent to use the propagate-uid-gid feature of the docker plugin. RUN groupadd -g 929 builder && useradd -m -s /bin/bash -u 929 -g 929 builder -USER builder -ADD set-python-version.sh /set-python-version.sh -RUN bash /set-python-version.sh ${PYTHON_VERSION} \ - && rm /set-python-version.sh +USER builder -#Packages required for running start-services inside the container -RUN yum install lsof -y +ADD set-python-version.sh /home/builder/set-python-version.sh +RUN bash /home/builder/set-python-version.sh 3.14 \ + && rm /home/builder/set-python-version.sh From f89156bafe6b23c232386c6fd1ea553b1cb2ecc5 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 11:22:08 +0200 Subject: [PATCH 37/58] chown Co-authored-by: Copilot --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0ecb46416..527522177 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,6 +14,6 @@ RUN groupadd -g 929 builder && useradd -m -s /bin/bash -u 929 -g 929 builder USER builder -ADD set-python-version.sh /home/builder/set-python-version.sh +ADD --chown builder:builder set-python-version.sh /home/builder/set-python-version.sh RUN bash /home/builder/set-python-version.sh 3.14 \ && rm /home/builder/set-python-version.sh From 0bf70ae8ba90326836a40dab5aea765b355798ce Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 11:23:54 +0200 Subject: [PATCH 38/58] fix --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 527522177..e23b2732c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,6 +14,6 @@ RUN groupadd -g 929 builder && useradd -m -s /bin/bash -u 929 -g 929 builder USER builder -ADD --chown builder:builder set-python-version.sh /home/builder/set-python-version.sh +ADD --chown=builder:builder set-python-version.sh /home/builder/set-python-version.sh RUN bash /home/builder/set-python-version.sh 3.14 \ && rm /home/builder/set-python-version.sh From 579cecca1d5cd49ad58008ce807dd39fe29fbb53 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 11:33:10 +0200 Subject: [PATCH 39/58] should work now --- .buildkite/pipeline.py | 16 ++++++++-------- docker/Dockerfile | 14 +++++++++++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index e283bb6b0..228c1945a 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -40,20 +40,20 @@ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( "linux-amd64-core2", - # "linux-aarch64", - # "windows-amd64-core2", - # "macos-aarch64", + "linux-aarch64", + "windows-amd64-core2", + "macos-aarch64", ) ] BUILD_TYPES = ["Release", "Debug"] PYTHON_VERSIONS = [ - # "3.9", - # "3.10", - # "3.11", - # "3.12", - # "3.13", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", "3.14", ] diff --git a/docker/Dockerfile b/docker/Dockerfile index e23b2732c..825d9092d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,14 +6,22 @@ FROM quay.io/pypa/$PLATFORM:${TAG} ARG PYTHON_VERSION +# We need to ensure that the builder user has the same UID and GID as the host buildkite agent to use the propagate-uid-gid feature of the docker plugin. +# The default UID, GID matches the default UID, GID of the buildkite agent on the host, if needed can be overridden +ARG USER_ID=929 +ARG GROUP_ID=929 +ARG USERNAME=builder +ARG HOME=/home/${USERNAME} +ARG COMMENT=builder + # Install packages required for running start-services inside the container RUN yum install lsof -y -# We need to ensure that the builder user has the same UID and GID as the host buildkite agent to use the propagate-uid-gid feature of the docker plugin. -RUN groupadd -g 929 builder && useradd -m -s /bin/bash -u 929 -g 929 builder +RUN groupadd --gid $GROUP_ID $USERNAME +RUN useradd --comment "$COMMENT" --home-dir $HOME --create-home --system --uid $USER_ID --gid $GROUP_ID $USERNAME USER builder ADD --chown=builder:builder set-python-version.sh /home/builder/set-python-version.sh -RUN bash /home/builder/set-python-version.sh 3.14 \ +RUN bash /home/builder/set-python-version.sh ${PYTHON_VERSION} \ && rm /home/builder/set-python-version.sh From 130ee488ea7e38fd35d1ba5e20a00e3905c2e1ed Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 11:39:11 +0200 Subject: [PATCH 40/58] comment Co-authored-by: Copilot --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 825d9092d..be8687ebe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ FROM quay.io/pypa/$PLATFORM:${TAG} ARG PYTHON_VERSION -# We need to ensure that the builder user has the same UID and GID as the host buildkite agent to use the propagate-uid-gid feature of the docker plugin. +# We need to ensure that the container user has the same UID and GID as the host buildkite agent to use the `propagate-uid-gid` feature of the docker plugin. # The default UID, GID matches the default UID, GID of the buildkite agent on the host, if needed can be overridden ARG USER_ID=929 ARG GROUP_ID=929 From 03a3b4af42d85c3509b47dec783e1d7aabef8205 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 11:43:00 +0200 Subject: [PATCH 41/58] remove debug prints --- scripts/teamcity/20.test.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/teamcity/20.test.sh b/scripts/teamcity/20.test.sh index 8edd21b30..ea3512f8b 100755 --- a/scripts/teamcity/20.test.sh +++ b/scripts/teamcity/20.test.sh @@ -10,15 +10,6 @@ set -u -x PYTHON="${PYTHON_CMD:-python3}" - -echo "Print User and Path info for debugging purposes" -echo "Current user: $(whoami)" -echo "Current user ID: $(id -u)" -echo "Current PATH: $PATH" -echo "Python executable: $(which ${PYTHON})" -echo "Python version: $(${PYTHON} --version)" - - ### # NOTE(leon): ### From af5208959d79ff61a2098f7ef25da78da0e67229 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 12:03:44 +0200 Subject: [PATCH 42/58] final check Co-authored-by: Copilot --- .buildkite/pipeline.py | 66 +++++------------------------------------- .buildkite/tools | 2 +- 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 228c1945a..f208f97d0 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -21,11 +21,13 @@ sys.path.insert(0, str(Path(__file__).parent / "tools")) from qdb_pipeline import ( Platform, - apply_docker, + apply_docker_compose, load_template, merge_env, select_platforms, validate_pipeline, + get_git_ref, + set_artifact_plugin_options, ) # noqa: E402 STEPS_DIR = Path(__file__).parent / "steps" @@ -99,58 +101,6 @@ def _env(p: Platform, step_name: str, build_type: str) -> dict[str, str]: ) -def _get_git_ref() -> str: - branch = os.environ.get("BUILDKITE_BRANCH") - tag = os.environ.get("BUILDKITE_TAG") - if not branch and not tag: - raise ValueError( - "BUILDKITE_BRANCH and BUILDKITE_TAG are both empty — are we running inside a Buildkite job?" - ) - - ref = f"refs/tags/{tag}" if tag else f"refs/heads/{branch}" - - return ref - -### Pending move to qdb-cicd-tools - -def _set_artifact_plugin_defaults( - step: dict, vars_per_step: dict[str, dict[str, str]] -) -> None: - """ - Goes through list of plugins and fills in defaults for qdb-artifacts plugin if present. - Doesn't overwrite existing keys, only fills in missing ones from provided dict. - """ - plugins = step.get("plugins", {}) - - for plugin_dict in plugins: - plugin_name = list(plugin_dict.keys())[0] - if plugin_name.startswith("bureau14/qdb-artifacts#"): - plugin_config = plugin_dict[plugin_name] - - for plugin_step, config in plugin_config.items(): - if plugin_step in vars_per_step.keys(): - for key in vars_per_step[plugin_step]: - if key not in config: - config[key] = vars_per_step[plugin_step][key] - - -def _apply_docker_compose( - step: dict, - config: dict[str, str] = {}, -) -> None: - DOCKER_COMPOSE_PLUGIN_VERSION = "v5.12.1" - - docker_plugin = { - f"docker-compose#{DOCKER_COMPOSE_PLUGIN_VERSION}": { - "propagate-uid-gid": True, - **config, - }, - } - existing = step.get("plugins", []) - step["plugins"] = [docker_plugin] + existing - -### - def _get_agent_python_env(platform: Platform, python_version: str) -> dict[str, str]: """ Returns environment variables to set for Python executable on the agent, based on platform and python version. @@ -187,15 +137,13 @@ def _apply_doc_command(step: dict, platform: Platform) -> None: def generate_pipeline() -> Pipeline: """Load templates, expand across platforms × build_types, overlay env and docker.""" pipeline = Pipeline() - git_ref = _get_git_ref() + git_ref = get_git_ref() group_steps = {} for p in PLATFORMS: for bt in BUILD_TYPES: for py in PYTHON_VERSIONS: - # TODO update slug logic in the submodule - slug = p.slug(bt.lower()) + f"-py{py.replace('.', '')}" - # slug = p.slug(bt.lower(), py.replace('.', '')) + slug = p.slug(bt.lower(), f"py{py.replace('.', '')}") # We want to use Release QuasarDB binaries when building Python API (debug and release) dependency_slug = p.slug("release") @@ -228,8 +176,8 @@ def generate_pipeline() -> Pipeline: env.update(_get_agent_python_env(p, py)) step["env"] = env if p.os == "linux": - _apply_docker_compose(step, compose_config) - _set_artifact_plugin_defaults(step, artifact_vars_per_step) + apply_docker_compose(step, compose_config) + set_artifact_plugin_options(step, artifact_vars_per_step) _apply_doc_command(step, p) # add step to group diff --git a/.buildkite/tools b/.buildkite/tools index 10a298957..476f355fa 160000 --- a/.buildkite/tools +++ b/.buildkite/tools @@ -1 +1 @@ -Subproject commit 10a29895745adb1e26094ed1f489d45923fecc96 +Subproject commit 476f355fa0955dcac43ff4fdd8b6675ce7280f77 From e2d972fea7c946b537504f78f6e9de57fb63bac0 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 12:11:57 +0200 Subject: [PATCH 43/58] signature changed --- .buildkite/pipeline.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index f208f97d0..b26b3581b 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -14,7 +14,6 @@ import dataclasses import sys from pathlib import Path -import os from buildkite_sdk import Pipeline, GroupStep @@ -42,9 +41,9 @@ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( "linux-amd64-core2", - "linux-aarch64", - "windows-amd64-core2", - "macos-aarch64", + # "linux-aarch64", + # "windows-amd64-core2", + # "macos-aarch64", ) ] @@ -148,9 +147,6 @@ def generate_pipeline() -> Pipeline: # We want to use Release QuasarDB binaries when building Python API (debug and release) dependency_slug = p.slug("release") - # TODO: this is just for testing, remove later - git_ref_dep = "refs/heads/sc-18547/buildkite" - # tvars = { "slug": slug, "queue": f"{p.queue_os}-{p.arch}", @@ -160,13 +156,13 @@ def generate_pipeline() -> Pipeline: artifact_vars_per_step = { "upload": {"variant": slug, "git-ref": git_ref}, "promote": {"variant": slug, "git-ref": git_ref}, - "download": {"variant": dependency_slug, "git-ref": git_ref_dep}, + "download": {"variant": dependency_slug, "git-ref": "refs/heads/sc-18547/buildkite"}, } compose_config = { "run": "pypa", "config": "docker/docker-compose.yml", - "propagate-environment": True, + "propagate-uid-gid": True, } step = load_template(STEPS_DIR / "_build.yml", **tvars) @@ -176,7 +172,7 @@ def generate_pipeline() -> Pipeline: env.update(_get_agent_python_env(p, py)) step["env"] = env if p.os == "linux": - apply_docker_compose(step, compose_config) + apply_docker_compose(step, config=compose_config) set_artifact_plugin_options(step, artifact_vars_per_step) _apply_doc_command(step, p) From 6bd48c350f21abb44a4c77bf130f5cf4940bc787 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 12:15:20 +0200 Subject: [PATCH 44/58] ready --- .buildkite/pipeline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index b26b3581b..234f3ee6c 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -41,9 +41,9 @@ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( "linux-amd64-core2", - # "linux-aarch64", - # "windows-amd64-core2", - # "macos-aarch64", + "linux-aarch64", + "windows-amd64-core2", + "macos-aarch64", ) ] @@ -156,7 +156,7 @@ def generate_pipeline() -> Pipeline: artifact_vars_per_step = { "upload": {"variant": slug, "git-ref": git_ref}, "promote": {"variant": slug, "git-ref": git_ref}, - "download": {"variant": dependency_slug, "git-ref": "refs/heads/sc-18547/buildkite"}, + "download": {"variant": dependency_slug, "git-ref": git_ref}, } compose_config = { From f731e7566fadf5e097bbffd8059d64a328f21147 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 27 Apr 2026 13:03:18 +0200 Subject: [PATCH 45/58] use image args --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index be8687ebe..5541f2f68 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,6 +22,6 @@ RUN useradd --comment "$COMMENT" --home-dir $HOME --create-home --system --uid $ USER builder -ADD --chown=builder:builder set-python-version.sh /home/builder/set-python-version.sh -RUN bash /home/builder/set-python-version.sh ${PYTHON_VERSION} \ - && rm /home/builder/set-python-version.sh +ADD --chown=$USERNAME:$USERNAME set-python-version.sh /home/$USERNAME/set-python-version.sh +RUN bash /home/${USERNAME}/set-python-version.sh ${PYTHON_VERSION} \ + && rm /home/${USERNAME}/set-python-version.sh From 191d94d8a8153cce2b0c4aa6dc2fac468f4dc56f Mon Sep 17 00:00:00 2001 From: Igor Date: Tue, 28 Apr 2026 09:07:21 +0200 Subject: [PATCH 46/58] Switch to correct user Co-authored-by: Copilot --- docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5541f2f68..85f3d3407 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,8 +20,8 @@ RUN yum install lsof -y RUN groupadd --gid $GROUP_ID $USERNAME RUN useradd --comment "$COMMENT" --home-dir $HOME --create-home --system --uid $USER_ID --gid $GROUP_ID $USERNAME -USER builder +USER $USERNAME -ADD --chown=$USERNAME:$USERNAME set-python-version.sh /home/$USERNAME/set-python-version.sh -RUN bash /home/${USERNAME}/set-python-version.sh ${PYTHON_VERSION} \ - && rm /home/${USERNAME}/set-python-version.sh +ADD --chown=$USERNAME:$USERNAME set-python-version.sh ${HOME}/set-python-version.sh +RUN bash ${HOME}/set-python-version.sh ${PYTHON_VERSION} \ + && rm ${HOME}/set-python-version.sh From 337198064a8b88e2561f31346055f83347830032 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 30 Apr 2026 10:20:31 +0200 Subject: [PATCH 47/58] env vars --- .buildkite/pipeline.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 234f3ee6c..7b7663e4b 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -105,17 +105,15 @@ def _get_agent_python_env(platform: Platform, python_version: str) -> dict[str, Returns environment variables to set for Python executable on the agent, based on platform and python version. Applies to Windows and macOS where we have multiple Python versions installed in different locations. """ - # XXX (igor) - # we can rely on referencing env variables instead of hardcoded paths but we need to update agents first to support this + python_version_slug = python_version.replace(".", "") if platform.os == "windows": return { - "PYTHON_EXECUTABLE": f"C:\\Python{python_version}-64\\python.exe", - "PYTHON_CMD": f"C:\\Python{python_version}-64\\python.exe", + "PYTHON_EXECUTABLE": f"$$QDB_CICD_AGENT_PYTHON_{python_version_slug}_64_EXE", + "PYTHON_CMD": f"$$QDB_CICD_AGENT_PYTHON_{python_version_slug}_64_EXE", } elif platform.os == "macos": return { - "PYTHON_EXECUTABLE": f"/opt/local/bin/python{python_version}", - "PYTHON_CMD": f"/opt/local/bin/python{python_version}", + "PYTHON_EXECUTABLE": f"$$QDB_CICD_AGENT_PYTHON_{python_version_slug}_PATH", } return {} From fbd539842760863d8afe0d5edb7294c3c7f15ef6 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 30 Apr 2026 10:23:08 +0200 Subject: [PATCH 48/58] test --- .buildkite/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 7b7663e4b..52c577921 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -40,8 +40,8 @@ PLATFORMS: list[Platform] = [ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( - "linux-amd64-core2", - "linux-aarch64", + # "linux-amd64-core2", + # "linux-aarch64", "windows-amd64-core2", "macos-aarch64", ) @@ -154,7 +154,7 @@ def generate_pipeline() -> Pipeline: artifact_vars_per_step = { "upload": {"variant": slug, "git-ref": git_ref}, "promote": {"variant": slug, "git-ref": git_ref}, - "download": {"variant": dependency_slug, "git-ref": git_ref}, + "download": {"variant": dependency_slug, "git-ref": "refs/heads/sc-18547/buildkite"}, } compose_config = { From 7d585afc31b03cd44461464404fa7ac646b3cdb2 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 30 Apr 2026 10:25:27 +0200 Subject: [PATCH 49/58] no retry in testing --- .buildkite/steps/_build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index a336c5bca..7a70bb120 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -1,9 +1,9 @@ agents: queue: "default-{queue}" -retry: - automatic: - limit: 3 +# retry: +# automatic: +# limit: 3 label: "{name} ({slug})" key: "build-{slug}" From ec0f1640d112e616547f1ad2803e896bcf50798f Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 30 Apr 2026 10:32:18 +0200 Subject: [PATCH 50/58] will work after rebuild --- .buildkite/pipeline.py | 4 ++-- .buildkite/steps/_build.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 52c577921..1c0aff531 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -40,8 +40,8 @@ PLATFORMS: list[Platform] = [ dataclasses.replace(p, **_OS_OVERLAY.get(p.os, {})) for p in select_platforms( - # "linux-amd64-core2", - # "linux-aarch64", + "linux-amd64-core2", + "linux-aarch64", "windows-amd64-core2", "macos-aarch64", ) diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index 7a70bb120..a336c5bca 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -1,9 +1,9 @@ agents: queue: "default-{queue}" -# retry: -# automatic: -# limit: 3 +retry: + automatic: + limit: 3 label: "{name} ({slug})" key: "build-{slug}" From 7eebc4cf4c326171477b86623ca3d7ef5f5e88a6 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:06:36 +0200 Subject: [PATCH 51/58] lint --- .buildkite/pipeline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 1c0aff531..d3018c75e 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -154,7 +154,10 @@ def generate_pipeline() -> Pipeline: artifact_vars_per_step = { "upload": {"variant": slug, "git-ref": git_ref}, "promote": {"variant": slug, "git-ref": git_ref}, - "download": {"variant": dependency_slug, "git-ref": "refs/heads/sc-18547/buildkite"}, + "download": { + "variant": dependency_slug, + "git-ref": git_ref, + }, } compose_config = { From 073d2f28035ec428606c6a21e20f80ac027dd876 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:24:37 +0200 Subject: [PATCH 52/58] add pre exit hook --- .buildkite/hooks/pre-exit | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .buildkite/hooks/pre-exit diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100644 index 000000000..0fab59ce0 --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,11 @@ +# This hook ensures that we don't leave qdbd cluster running even if one of the steps in the pipeline fails. +# It will stop the cluster if it is running, and ignore any errors that may occur during the stopping process. + +SCRIPT_PATH="scripts/tests/setup/stop-services.sh" + +if [ -f "$SCRIPT_PATH" ]; then + bash "$SCRIPT_PATH" > /dev/null 2>&1 || true + echo "Stopped services" +fi + +exit 0 From 2cb3af6f39289e7f91700c62e4e5f6bb2c42c4af Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:27:58 +0200 Subject: [PATCH 53/58] Add symlink from scripts/teamcity to scripts/cicd --- .buildkite/steps/_build.yml | 4 ++-- scripts/cicd/00.common.sh | 1 + scripts/cicd/10.build.sh | 1 + scripts/cicd/20.test.sh | 1 + scripts/cicd/30.doc.sh | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) create mode 120000 scripts/cicd/00.common.sh create mode 120000 scripts/cicd/10.build.sh create mode 120000 scripts/cicd/20.test.sh create mode 120000 scripts/cicd/30.doc.sh diff --git a/.buildkite/steps/_build.yml b/.buildkite/steps/_build.yml index a336c5bca..10702b062 100644 --- a/.buildkite/steps/_build.yml +++ b/.buildkite/steps/_build.yml @@ -12,11 +12,11 @@ commands: - echo \"+++ Start Services\" - bash scripts/tests/setup/start-services.sh - echo \"+++ Run Tests\" - - bash scripts/teamcity/20.test.sh + - bash scripts/cicd/20.test.sh - echo \"+++ Stop Services\" - bash scripts/tests/setup/stop-services.sh - echo \"+++ Run build\" - - bash scripts/teamcity/10.build.sh + - bash scripts/cicd/10.build.sh plugins: - bureau14/qdb-artifacts#master: diff --git a/scripts/cicd/00.common.sh b/scripts/cicd/00.common.sh new file mode 120000 index 000000000..14b8dd2fd --- /dev/null +++ b/scripts/cicd/00.common.sh @@ -0,0 +1 @@ +teamcity/00.common.sh \ No newline at end of file diff --git a/scripts/cicd/10.build.sh b/scripts/cicd/10.build.sh new file mode 120000 index 000000000..8c54afa50 --- /dev/null +++ b/scripts/cicd/10.build.sh @@ -0,0 +1 @@ +teamcity/10.build.sh \ No newline at end of file diff --git a/scripts/cicd/20.test.sh b/scripts/cicd/20.test.sh new file mode 120000 index 000000000..2255e8da7 --- /dev/null +++ b/scripts/cicd/20.test.sh @@ -0,0 +1 @@ +teamcity/20.test.sh \ No newline at end of file diff --git a/scripts/cicd/30.doc.sh b/scripts/cicd/30.doc.sh new file mode 120000 index 000000000..4362d826a --- /dev/null +++ b/scripts/cicd/30.doc.sh @@ -0,0 +1 @@ +teamcity/30.doc.sh \ No newline at end of file From c47b569e87e81a4e6af93c2343bd28db5f2d39c8 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:28:25 +0200 Subject: [PATCH 54/58] last test --- .buildkite/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index d3018c75e..2ba3bd44b 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -156,7 +156,7 @@ def generate_pipeline() -> Pipeline: "promote": {"variant": slug, "git-ref": git_ref}, "download": { "variant": dependency_slug, - "git-ref": git_ref, + "git-ref": "refs/heads/sc-18547/buildkite", }, } From adb6b0586bd651bdd423232c594958a6dd18f226 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:37:34 +0200 Subject: [PATCH 55/58] fix symlinks --- scripts/cicd/00.common.sh | 2 +- scripts/cicd/10.build.sh | 2 +- scripts/cicd/20.test.sh | 2 +- scripts/cicd/30.doc.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cicd/00.common.sh b/scripts/cicd/00.common.sh index 14b8dd2fd..92a168020 120000 --- a/scripts/cicd/00.common.sh +++ b/scripts/cicd/00.common.sh @@ -1 +1 @@ -teamcity/00.common.sh \ No newline at end of file +../teamcity/00.common.sh \ No newline at end of file diff --git a/scripts/cicd/10.build.sh b/scripts/cicd/10.build.sh index 8c54afa50..79b5dcbc7 120000 --- a/scripts/cicd/10.build.sh +++ b/scripts/cicd/10.build.sh @@ -1 +1 @@ -teamcity/10.build.sh \ No newline at end of file +../teamcity/10.build.sh \ No newline at end of file diff --git a/scripts/cicd/20.test.sh b/scripts/cicd/20.test.sh index 2255e8da7..0023c9ccc 120000 --- a/scripts/cicd/20.test.sh +++ b/scripts/cicd/20.test.sh @@ -1 +1 @@ -teamcity/20.test.sh \ No newline at end of file +../teamcity/20.test.sh \ No newline at end of file diff --git a/scripts/cicd/30.doc.sh b/scripts/cicd/30.doc.sh index 4362d826a..06fe807d3 120000 --- a/scripts/cicd/30.doc.sh +++ b/scripts/cicd/30.doc.sh @@ -1 +1 @@ -teamcity/30.doc.sh \ No newline at end of file +../teamcity/30.doc.sh \ No newline at end of file From a0a30504d84ec2fb8baab6730175a24b383aaab9 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:43:46 +0200 Subject: [PATCH 56/58] remove symlink --- scripts/cicd/00.common.sh | 1 - scripts/cicd/10.build.sh | 1 - scripts/cicd/20.test.sh | 1 - scripts/cicd/30.doc.sh | 1 - 4 files changed, 4 deletions(-) delete mode 120000 scripts/cicd/00.common.sh delete mode 120000 scripts/cicd/10.build.sh delete mode 120000 scripts/cicd/20.test.sh delete mode 120000 scripts/cicd/30.doc.sh diff --git a/scripts/cicd/00.common.sh b/scripts/cicd/00.common.sh deleted file mode 120000 index 92a168020..000000000 --- a/scripts/cicd/00.common.sh +++ /dev/null @@ -1 +0,0 @@ -../teamcity/00.common.sh \ No newline at end of file diff --git a/scripts/cicd/10.build.sh b/scripts/cicd/10.build.sh deleted file mode 120000 index 79b5dcbc7..000000000 --- a/scripts/cicd/10.build.sh +++ /dev/null @@ -1 +0,0 @@ -../teamcity/10.build.sh \ No newline at end of file diff --git a/scripts/cicd/20.test.sh b/scripts/cicd/20.test.sh deleted file mode 120000 index 0023c9ccc..000000000 --- a/scripts/cicd/20.test.sh +++ /dev/null @@ -1 +0,0 @@ -../teamcity/20.test.sh \ No newline at end of file diff --git a/scripts/cicd/30.doc.sh b/scripts/cicd/30.doc.sh deleted file mode 120000 index 06fe807d3..000000000 --- a/scripts/cicd/30.doc.sh +++ /dev/null @@ -1 +0,0 @@ -../teamcity/30.doc.sh \ No newline at end of file From 52c2842e5717f8daf8f77c0acc5af21a52390948 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:44:54 +0200 Subject: [PATCH 57/58] add scripts --- scripts/cicd/00.common.sh | 6 ++ scripts/cicd/10.build.sh | 56 +++++++++++++ scripts/cicd/20.test.sh | 170 ++++++++++++++++++++++++++++++++++++++ scripts/cicd/30.doc.sh | 34 ++++++++ 4 files changed, 266 insertions(+) create mode 100755 scripts/cicd/00.common.sh create mode 100755 scripts/cicd/10.build.sh create mode 100755 scripts/cicd/20.test.sh create mode 100755 scripts/cicd/30.doc.sh diff --git a/scripts/cicd/00.common.sh b/scripts/cicd/00.common.sh new file mode 100755 index 000000000..4b5cf38b7 --- /dev/null +++ b/scripts/cicd/00.common.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +if [ -f $HOME/.bashrc ] +then + source $HOME/.bashrc +fi diff --git a/scripts/cicd/10.build.sh b/scripts/cicd/10.build.sh new file mode 100755 index 000000000..38663e4e2 --- /dev/null +++ b/scripts/cicd/10.build.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null && pwd)" + +source ${SCRIPT_DIR}/00.common.sh + +git config --global --add safe.directory '*' + +# No more errors should occur after here +set -e -u -x + +PYTHON="${PYTHON_CMD:-python3}" + +# Now use a virtualenv to run the tests. If the virtualenv already exists, we remove +# it to ensure a clean install. +${PYTHON} -m venv --clear ${SCRIPT_DIR}/../../.env/ +if [[ "$(uname)" == MINGW* ]] +then + VENV_PYTHON="${SCRIPT_DIR}/../../.env/Scripts/python.exe" +else + VENV_PYTHON="${SCRIPT_DIR}/../../.env/bin/python" +fi + +${VENV_PYTHON} --version + +function relabel_wheel { + wheel="$1" + + if ! ${VENV_PYTHON} -m auditwheel show "$wheel" + then + echo "Skipping non-platform specific wheel $wheel" + else + # ${AUDITWHEEL_PLAT} is defined in manylinux base docker image + ${VENV_PYTHON} -m auditwheel repair "$wheel" --plat "$AUDITWHEEL_PLAT" -w dist/ + rm "$wheel" + fi +} + +rm -r -f build/ dist/ + +if [[ "$OSTYPE" == "darwin"* && $PYTHON == "python3.9"* ]]; then + ${VENV_PYTHON} -m pip install --upgrade setuptools==63.0.0b1 wheel +else + ${VENV_PYTHON} -m pip install --upgrade setuptools wheel auditwheel +fi + +${VENV_PYTHON} -m pip install -r dev-requirements.txt + +export DISTUTILS_DEBUG=1 +export QDB_TESTS_ENABLED=OFF + +${VENV_PYTHON} -m build -w + +for whl in dist/*.whl; do + relabel_wheel "$whl" +done diff --git a/scripts/cicd/20.test.sh b/scripts/cicd/20.test.sh new file mode 100755 index 000000000..ea3512f8b --- /dev/null +++ b/scripts/cicd/20.test.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null && pwd)" + +source ${SCRIPT_DIR}/00.common.sh + +git config --global --add safe.directory '*' + +set -u -x + +PYTHON="${PYTHON_CMD:-python3}" + +### +# NOTE(leon): +### +# +# EVIL CODE SECTION BELOW. +# +# Looking at this will make you angry. +# Studying it will make you cry. +# Understanding it will bring enlightenment. +# Using it will enable massive parallelism for builds on Windows by using Ninja. +# +# ~~~ +# +# The problem is solves: we want to use Ninja for Windows. For this, we need to set a whole +# bunch of environment variables. Microsoft helpfully provided a batch script called 'VsDevCmd.bat' +# which does exactly this. +# +# But herein lies the problem: we cannot just "source" a windows batch script from mingw/bash. +# We can use cmd to invoke it, but then it exits and the environment variables are immediately lost. +# +# The evil code below solves this by effectively invoking the VsDevCmd.bat, and then storing the +# entire environment in a file. Then we read this environment file, and export all the env keys. +# +# `evil_inner` is responsible for invoking the VsDevCmd and printing out all the environment +# variables to the console. +# +# `evil_outer` is responsible for capturing environment variables before, after, and setting all +# the changed variables. + +function evil_inner { + local arch=$1; shift; + + # Because VsDevCmd.bat prints some garbage data, we want to skip the first X rows. Basically + # what I did was audit manually the type of regex that actually matches all environment data. + # Then, because we know 100% sure after the first row is matched, the rest is also going to + # be from printenv, we just tell grep "print the 5000 lines after your first match". This + # avoids a scenario that if for some stupid reason a new env var is introduced which doesn't + # match the regex, it still works. + + (cd /c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2022/BuildTools/Common7/Tools/ \ + && cmd //C "VsDevCmd.bat -host_arch=amd64 -arch=$arch && bash -c printenv" \ + | grep -A5000 -m1 -E '^([a-zA-Z0-9_\(\):]+)=' ) +} + +function evil_outer { + local arch=$1; shift; + + before=$(mktemp) + after=$(mktemp) + added=$(mktemp) + + printenv | sort > $before + evil_inner $arch | sort > $after + + # Keep only those lines that are added/different in the $after file, and save them in a file. + # Yes, I too did not know what `comm` was until I needed it, and it basically compares two sorted + # files line-by-line, which is exactly what we need here. + + comm -13 $before $after > $added + + while read -r LINE; do + + local key=$(cut -d '=' -f 1 <<< "$LINE" ) + local value=$(cut -d '=' -f 2- <<< "$LINE" ) + + ### + # XXX(leon): + ### + # + # It seems MSVC sets a few very weird environment variables with keys: + # - !C:=C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools + # - !ExitCode=00000000 + # + # I have no idea how to properly export these, but it doesn't appear to matter a lot. I *suspect* + # that they're really some internal environment variables that represent the current working dir + # and/or exit code of a program, and are not intended to be set. + # + # However, this will print out warnings. These can be safely ignored. + + export ${key}="${value}" + done < $added +} + +### +# +# EVIL CODE SECTION ABOVE. NO MORE EVIL CODE BELOW THIS LINE. +# +### + +if [[ "$(uname)" == MINGW* ]] +then + ARCH_BITS=$(${PYTHON} -c 'import struct;print( 8 * struct.calcsize("P"))') + echo "Windows build detected, target arch with bits: ${ARCH_BITS}" + + if [[ "${ARCH_BITS}" == "32" ]] + then + echo "Targeting win32" + evil_outer x86 + elif [[ "${ARCH_BITS}" == "64" ]] + then + echo "Targeting win64" + evil_outer amd64 + else + echo "Internal error: 'ARCH_BITS' is unrecognized: ${ARCH_BITS}" + exit -1 + fi +fi + +# No more errors should occur after here +set -e -o pipefail + +if [[ -d "dist/" ]] +then + echo "Removing dist/" + rm -rf dist/ +fi + +if [[ -d "build/" ]] +then + echo "Warning: build/ directory already exists, assuming incremental compilation, reusing build artifacts" +fi + +# Now use a virtualenv to run the tests. If the virtualenv already exists, we reuse it +# to avoid frequent rebuilds. + +if [[ -d "${SCRIPT_DIR}/../../.env/" ]] +then + echo "virtualenv already exists, skip creating new venv" +else + echo "Creating new virtualenv" + ${PYTHON} -m venv --clear ${SCRIPT_DIR}/../../.env/ +fi + +if [[ "$(uname)" == MINGW* ]] +then + VENV_PYTHON="${SCRIPT_DIR}/../../.env/Scripts/python.exe" +else + VENV_PYTHON="${SCRIPT_DIR}/../../.env/bin/python" +fi + +${VENV_PYTHON} -m pip install --upgrade -r dev-requirements.txt + +export QDB_TESTS_ENABLED=ON +${VENV_PYTHON} -m build -w + +${VENV_PYTHON} -m pip install --no-deps --force-reinstall dist/quasardb-*.whl + +echo "Invoking pytest" + +TEST_OPTS="$@" +if [[ ! -z ${JUNIT_XML_FILE-} ]] +then + TEST_OPTS+=" --junitxml=${JUNIT_XML_FILE}" +fi + +pushd tests +exec ${VENV_PYTHON} -m pytest ${TEST_OPTS} +popd diff --git a/scripts/cicd/30.doc.sh b/scripts/cicd/30.doc.sh new file mode 100755 index 000000000..526de99ba --- /dev/null +++ b/scripts/cicd/30.doc.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null && pwd)" + +source ${SCRIPT_DIR}/00.common.sh + +set -u -x + +PYTHON="${PYTHON_CMD:-python3}" + +${PYTHON} -m venv --clear ${SCRIPT_DIR}/../../.env/ +if [[ "$(uname)" == MINGW* ]] +then + VENV_PYTHON="${SCRIPT_DIR}/../../.env/Scripts/python.exe" +else + VENV_PYTHON="${SCRIPT_DIR}/../../.env/bin/python" +fi + +${VENV_PYTHON} -m pip install -r dev-requirements.txt +${VENV_PYTHON} -m pip install --no-deps --force-reinstall dist/quasardb-*manylinux*.whl +${VENV_PYTHON} -m pip install --upgrade pdoc3==0.11.5 + + +# To avoid conflicts with `quasardb` directory and `import quasardb` +rm -rf doc/build || true + +mkdir doc/build +pushd doc + +${VENV_PYTHON} docgen.py + +popd + +tar -czvf dist/doc.tar.gz -C doc/build . From ec5243e56ca5466050bcefe3fbc7bd41d73776fa Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 4 May 2026 08:55:20 +0200 Subject: [PATCH 58/58] done --- .buildkite/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 2ba3bd44b..d3018c75e 100644 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -156,7 +156,7 @@ def generate_pipeline() -> Pipeline: "promote": {"variant": slug, "git-ref": git_ref}, "download": { "variant": dependency_slug, - "git-ref": "refs/heads/sc-18547/buildkite", + "git-ref": git_ref, }, }