Skip to content

Source PQS dashboard version from Scribe annotation #936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
danielporterda wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Source PQS dashboard version from Scribe annotation #936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 65 additions & 25 deletions scripts/generate_network_component_versions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,21 @@
PQS_IMAGE_REPOSITORY = (
"europe-docker.pkg.dev/da-images/public/docker/participant-query-store"
)
PQS_TAGS_URL = (
"https://europe-docker.pkg.dev/v2/da-images/public/docker/"
"participant-query-store/tags/list"
PQS_SCRIBE_COMPONENT_REPOSITORY = (
"europe-docker.pkg.dev/da-images/public/components/scribe"
)
PQS_SCRIBE_RELEASE_LINE_TAG = "3.5"
PQS_SCRIBE_MANIFEST_URL = (
"https://europe-docker.pkg.dev/v2/da-images/public/components/"
f"scribe/manifests/{PQS_SCRIBE_RELEASE_LINE_TAG}"
)
OCI_MANIFEST_ACCEPT = ", ".join(
[
"application/vnd.oci.image.index.v1+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v2+json",
]
)
STABLE_SEMVER_RE = re.compile(r"\d+\.\d+\.\d+")
USER_AGENT = "cf-docs-version-dashboard-generator"
Expand Down Expand Up @@ -140,6 +152,14 @@ def fetch_json(url: str, timeout: float) -> dict:
return json.load(response)


def fetch_manifest_json(url: str, timeout: float) -> dict:
headers = request_headers(url)
headers["Accept"] = OCI_MANIFEST_ACCEPT
request = Request(url, headers=headers)
with urlopen(request, timeout=timeout) as response:
return json.load(response)


def fetch_text(url: str, timeout: float) -> str:
with request_url(url, timeout) as response:
return response.read().decode("utf-8", "replace")
Expand Down Expand Up @@ -250,20 +270,33 @@ def fetch_latest_wallet_gateway_version(timeout: float) -> str:
return latest_stable_version(versions, WALLET_GATEWAY_RELEASES_URL)


def fetch_latest_pqs_version(timeout: float) -> str:
data = fetch_json(PQS_TAGS_URL, timeout)
tags: list[str] = []
manifest = data.get("manifest", {})
if not isinstance(manifest, dict):
raise RuntimeError(f"Expected Artifact Registry manifest map from {PQS_TAGS_URL}")
for entry in manifest.values():
if not isinstance(entry, dict):
continue
entry_tags = entry.get("tag", [])
if not isinstance(entry_tags, list):
continue
tags.extend(tag for tag in entry_tags if isinstance(tag, str))
return latest_stable_version(tags, PQS_TAGS_URL)
def previous_stable_pqs_version(existing_config: dict) -> str:
versions = [
existing_repo_version(existing_config, "pqs", network_key)
for network_key in NETWORK_ORDER
]
return latest_stable_version(versions, "existing PQS dashboard config")


def fetch_pqs_version_from_scribe_component(
timeout: float,
*,
previous_stable_version: str,
) -> str:
data = fetch_manifest_json(PQS_SCRIBE_MANIFEST_URL, timeout)
annotations = data.get("annotations", {})
if not isinstance(annotations, dict):
raise RuntimeError(f"Expected manifest annotations from {PQS_SCRIBE_MANIFEST_URL}")
version = str(
annotations.get("org.opencontainers.image.version")
or annotations.get("com.digitalasset.version")
or ""
)
if not version:
raise RuntimeError(f"Missing Scribe image version annotation in {PQS_SCRIBE_MANIFEST_URL}")
if STABLE_SEMVER_RE.fullmatch(version):
return version
return previous_stable_version


def clean_html_text(value: str) -> str:
Expand Down Expand Up @@ -433,17 +466,22 @@ def collect_network_snapshot(network_key: str, timeout: float) -> dict:
}


def collect_snapshot(timeout: float) -> dict:
def collect_snapshot(timeout: float, existing_config: dict) -> dict:
generated_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
latest_dpm_sdk = fetch_text(DPM_LATEST_URL, timeout).strip()
previous_pqs = previous_stable_pqs_version(existing_config)
return {
"generatedAt": generated_at,
"generatorMode": "public_source_collection_with_manual_fallbacks",
"networks": {
network_key: collect_network_snapshot(network_key, timeout)
for network_key in NETWORK_ORDER
},
"latestDpmSdk": fetch_text(DPM_LATEST_URL, timeout).strip(),
"latestPqs": fetch_latest_pqs_version(timeout),
"latestDpmSdk": latest_dpm_sdk,
"latestPqs": fetch_pqs_version_from_scribe_component(
timeout,
previous_stable_version=previous_pqs,
),
"latestWalletGateway": fetch_latest_wallet_gateway_version(timeout),
"npmVersions": {
key: fetch_npm_latest(package_name, timeout)
Expand Down Expand Up @@ -484,7 +522,7 @@ def repository_url(repository_key: str, existing_config: dict) -> str:
if repository_key == "damlSdk":
return SPLICE_REPOSITORY_URL
if repository_key == "pqs":
return f"https://{PQS_IMAGE_REPOSITORY}"
return f"https://{PQS_SCRIBE_COMPONENT_REPOSITORY}"
if repository_key == "walletGateway":
return WALLET_GATEWAY_RELEASES_PAGE_URL
if repository_key in NPM_PACKAGE_URLS:
Expand Down Expand Up @@ -512,7 +550,7 @@ def build_repository_mapping(
elif repository_key == "pqs":
external_version = snapshot["latestPqs"]
branch = ""
folder_path_repo = PQS_IMAGE_REPOSITORY
folder_path_repo = f"{PQS_SCRIBE_COMPONENT_REPOSITORY}:{PQS_SCRIBE_RELEASE_LINE_TAG}"
elif repository_key in NPM_PACKAGE_NAMES:
external_version = snapshot["npmVersions"][repository_key]
branch = ""
Expand Down Expand Up @@ -581,8 +619,10 @@ def build_source_contract(snapshot: dict) -> dict:
f"{WALLET_GATEWAY_RELEASE_REPO}."
),
"pqs": (
"Latest stable semver tag from the public Artifact Registry image "
f"{PQS_IMAGE_REPOSITORY}."
"Read org.opencontainers.image.version from the public Artifact Registry "
f"Scribe component image {PQS_SCRIBE_COMPONENT_REPOSITORY}:"
f"{PQS_SCRIBE_RELEASE_LINE_TAG}. If the floating release-line tag resolves "
"to a prerelease, retain the previous stable dashboard value."
),
"minProtocolVersion": "Manual/fallback until a public live source is identified.",
"darVersions": (
Expand Down Expand Up @@ -630,7 +670,7 @@ def run_helper() -> None:
def main() -> int:
args = parse_args()
existing_config = read_existing_config(DEFAULT_REPO_CONFIG_OUTPUT)
snapshot = collect_snapshot(args.timeout)
snapshot = collect_snapshot(args.timeout, existing_config)
repo_version_config = build_config(existing_config, snapshot)

if args.dry_run:
Expand Down
94 changes: 94 additions & 0 deletions tests/test_generate_network_component_versions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,100 @@ def test_latest_stable_version_ignores_prerelease_and_debug_tags() -> None:
)


def test_previous_stable_pqs_version_uses_existing_dashboard_config() -> None:
module = load_script_module()

assert (
module.previous_stable_pqs_version(
{
"repositories": {
"pqs": {
"versionMapping": {
"mainnet": {"externalVersion": "3.5.1"},
"testnet": {"externalVersion": "3.5.2"},
"devnet": {"externalVersion": "3.5.2-rc1"},
}
}
}
}
)
== "3.5.2"
)


def test_fetch_pqs_version_from_scribe_component_uses_stable_annotation(monkeypatch) -> None:
module = load_script_module()

def fake_fetch_manifest_json(url: str, timeout: float) -> dict:
assert url == module.PQS_SCRIBE_MANIFEST_URL
return {
"annotations": {
"org.opencontainers.image.version": "3.5.3",
"com.digitalasset.version": "3.5.3",
}
}

monkeypatch.setattr(module, "fetch_manifest_json", fake_fetch_manifest_json)

assert (
module.fetch_pqs_version_from_scribe_component(
timeout=1.0,
previous_stable_version="3.5.2",
)
== "3.5.3"
)


def test_fetch_pqs_version_from_scribe_component_keeps_previous_stable_for_prerelease(
monkeypatch,
) -> None:
module = load_script_module()

def fake_fetch_manifest_json(url: str, timeout: float) -> dict:
assert url == module.PQS_SCRIBE_MANIFEST_URL
return {
"annotations": {
"org.opencontainers.image.version": "3.5.3-rc2",
"com.digitalasset.version": "3.5.3-rc2",
}
}

monkeypatch.setattr(module, "fetch_manifest_json", fake_fetch_manifest_json)

assert (
module.fetch_pqs_version_from_scribe_component(
timeout=1.0,
previous_stable_version="3.5.2",
)
== "3.5.2"
)


def test_build_config_records_pqs_scribe_component_source() -> None:
module = load_script_module()

config = module.build_config(
{"versions": {}, "repositories": {}},
dashboard_snapshot(
generated_at="2026-06-03T12:00:00+00:00",
splice_version="0.6.3",
),
)

assert config["repositories"]["pqs"]["url"] == (
f"https://{module.PQS_SCRIBE_COMPONENT_REPOSITORY}"
)
assert config["repositories"]["pqs"]["versionMapping"]["mainnet"] == {
"branch": "",
"externalVersion": "3.5.1",
"folderPathRepo": (
f"{module.PQS_SCRIBE_COMPONENT_REPOSITORY}:"
f"{module.PQS_SCRIBE_RELEASE_LINE_TAG}"
),
}
assert "org.opencontainers.image.version" in config["_generated"]["sourceContract"]["pqs"]


def test_request_headers_use_github_token_for_github_api(monkeypatch) -> None:
module = load_script_module()
monkeypatch.setenv("GITHUB_TOKEN", "test-token")
Expand Down