Draft
Conversation
daryaguettler
force-pushed
the
feature/improve-visualizations
branch
from
ba62c85 to
1a7a4ba
Compare
add export functionality for d3 visualizations update run ordering and colorschemes remove setuptools from pyproject update pyproject to align with main
daryaguettler
force-pushed
the
feature/improve-visualizations
branch
from
1a7a4ba to
db5bab4
Compare
szvsw
commented
Feb 23, 2026
Comment on lines
+46
to
+148
| def _get_s3_prefixes( | ||
| s3_client: S3Client, | ||
| bucket: str, | ||
| prefix: str, | ||
| ) -> list[str]: | ||
| """List all common prefixes under a given S3 prefix.""" | ||
| paginator = s3_client.get_paginator("list_objects_v2") | ||
| prefixes: list[str] = [] | ||
| for page in paginator.paginate( | ||
| Bucket=bucket, | ||
| Prefix=prefix, | ||
| Delimiter="/", | ||
| PaginationConfig={"PageSize": 1000}, | ||
| ): | ||
| for common_prefix in page.get("CommonPrefixes", []): | ||
| p = common_prefix.get("Prefix", "") | ||
| if p: | ||
| prefixes.append(p) | ||
| return prefixes | ||
|
|
||
|
|
||
| def _extract_experiment_names(prefixes: list[str], base_prefix: str) -> list[str]: | ||
| """Extract experiment names from S3 prefixes.""" | ||
| names = [] | ||
| for p in prefixes: | ||
| name = p[len(base_prefix) :].rstrip("/") | ||
| if name: | ||
| names.append(name) | ||
| return names | ||
|
|
||
|
|
||
| def _extract_versions(prefixes: list[str], exp_prefix: str) -> list[str]: | ||
| """Extract version strings from S3 prefixes.""" | ||
| versions = [] | ||
| for p in prefixes: | ||
| version = p[len(exp_prefix) :].rstrip("/") | ||
| if version.startswith("v"): | ||
| versions.append(version) | ||
| return versions | ||
|
|
||
|
|
||
| def list_s3_experiments( | ||
| bucket: str | None = None, | ||
| prefix: str | None = None, | ||
| s3_client: S3Client | None = None, | ||
| ) -> list[S3ExperimentInfo]: | ||
| """List all available experiments in S3. | ||
|
|
||
| Args: | ||
| bucket: S3 bucket name. If None, uses ScytheStorageSettings. | ||
| prefix: S3 prefix. If None, uses ScytheStorageSettings.BUCKET_PREFIX. | ||
| s3_client: Optional S3 client. If None, creates a new one. | ||
|
|
||
| Returns: | ||
| List of S3ExperimentInfo objects with experiment names and versions. | ||
| """ | ||
| import boto3 | ||
| from scythe.settings import ScytheStorageSettings | ||
|
|
||
| settings = ScytheStorageSettings() | ||
| bucket = bucket or settings.BUCKET | ||
| prefix = prefix or settings.BUCKET_PREFIX | ||
|
|
||
| if s3_client is None: | ||
| s3_client = boto3.client("s3") | ||
|
|
||
| if not prefix.endswith("/"): | ||
| prefix = prefix + "/" | ||
|
|
||
| exp_prefixes = _get_s3_prefixes(s3_client, bucket, prefix) | ||
| exp_names = _extract_experiment_names(exp_prefixes, prefix) | ||
|
|
||
| result = [] | ||
| for exp_name in sorted(exp_names): | ||
| exp_prefix = f"{prefix}{exp_name}/" | ||
| version_prefixes = _get_s3_prefixes(s3_client, bucket, exp_prefix) | ||
| versions = _extract_versions(version_prefixes, exp_prefix) | ||
| sorted_versions = _sort_versions(versions) | ||
| latest = sorted_versions[-1] if sorted_versions else None | ||
| result.append( | ||
| S3ExperimentInfo( | ||
| run_name=exp_name, | ||
| versions=sorted_versions, | ||
| latest_version=latest, | ||
| ) | ||
| ) | ||
| return result | ||
|
|
||
|
|
||
| def _sort_versions(versions: list[str]) -> list[str]: | ||
| """Sort semantic versions in ascending order.""" | ||
|
|
||
| def parse_version(v: str) -> tuple[int, int, int]: | ||
| if v.startswith("v"): | ||
| v = v[1:] | ||
| parts = v.replace("-", ".").split(".") | ||
| return ( | ||
| int(parts[0]) if len(parts) > 0 else 0, | ||
| int(parts[1]) if len(parts) > 1 else 0, | ||
| int(parts[2]) if len(parts) > 2 else 0, | ||
| ) | ||
|
|
||
| return sorted(versions, key=parse_version) |
Contributor
Author
There was a problem hiding this comment.
Is there a reason this is not using the equivalent code from Scythe?
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
add export functionality for d3 visualizations
update run ordering and colorschemes
remove setuptools from pyproject