From 56dd7bf94020ee0afca4fa7ab56ca68b7a7671a5 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 16:58:10 +0900 Subject: [PATCH 01/89] feat: improve usage table readability --- result_server/templates/usage_report.html | 69 +++++++++++++++++++++-- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/result_server/templates/usage_report.html b/result_server/templates/usage_report.html index 7e79fda..82293f8 100644 --- a/result_server/templates/usage_report.html +++ b/result_server/templates/usage_report.html @@ -9,15 +9,39 @@ @@ -35,6 +75,7 @@

Meta Information

+
@@ -47,17 +88,20 @@

Meta Information

Code{{ result.get('code', 'N/A') }}
Exp{{ result.get('exp', 'N/A') }}
Estimate Timestamp{{ estimate_meta.get('estimation_result_timestamp', 'N/A') }}
Performance Ratio{{ "%.3f"|format(result.get('performance_ratio', 'N/A')|float) if result.get('performance_ratio') not in [None, '', 'N/A', 'null', 'nan'] else result.get('performance_ratio', 'N/A') }}
+

Package Resolution

+
Current Requested{{ estimate_meta.get('current_package', {}).get('requested_estimation_package', 'N/A') }}
Current Applied{{ estimate_meta.get('current_package', {}).get('estimation_package', 'N/A') }}
Future Requested{{ estimate_meta.get('future_package', {}).get('requested_estimation_package', 'N/A') }}
Future Applied{{ estimate_meta.get('future_package', {}).get('estimation_package', 'N/A') }}
+

Measurement

{{ measurement | tojson(indent=2) }}
@@ -68,6 +112,7 @@

Confidence

Current System

+
@@ -81,10 +126,12 @@

Current System

System{{ current.get('system', 'N/A') }}
FOM{{ current.get('fom', 'N/A') }}
Model{{ current.get('model', {}).get('name', 'N/A') }}
Model Type{{ current.get('model', {}).get('type', 'N/A') }}
+

Future System

+
@@ -98,6 +145,7 @@

Future System

System{{ future.get('system', 'N/A') }}
FOM{{ future.get('fom', 'N/A') }}
Model{{ future.get('model', {}).get('name', 'N/A') }}
Model Type{{ future.get('model', {}).get('type', 'N/A') }}
+
@@ -112,6 +160,7 @@

Current Breakdown

{% if current_breakdown.get('sections') or current_breakdown.get('overlaps') %} {% if current_breakdown.get('sections') %}

Sections

+
@@ -125,7 +174,7 @@

Sections

- +
NameTimePackageScalingFallbackApplicability
{{ item.get('time', 'N/A') }} {{ item.get('estimation_package', 'N/A') }} {{ item.get('scaling_method', 'N/A') }}{{ item.get('fallback_used', '—') }}{{ item.get('fallback_used', '—') }} {% if applicability %}
{{ applicability.get('status', 'N/A') }}
@@ -140,9 +189,11 @@

Sections

{% endfor %}
+
{% endif %} {% if current_breakdown.get('overlaps') %}

Overlaps

+
@@ -156,7 +207,7 @@

Overlaps

- +
SectionsTimePackageScalingFallbackApplicability
{{ item.get('time', 'N/A') }} {{ item.get('estimation_package', 'N/A') }} {{ item.get('scaling_method', 'N/A') }}{{ item.get('fallback_used', '—') }}{{ item.get('fallback_used', '—') }} {% if applicability %}
{{ applicability.get('status', 'N/A') }}
@@ -171,6 +222,7 @@

Overlaps

{% endfor %}
+
{% endif %} {% else %}
No section or overlap breakdown is stored for the current-side estimate.
@@ -182,6 +234,7 @@

Future Breakdown

{% if future_breakdown.get('sections') or future_breakdown.get('overlaps') %} {% if future_breakdown.get('sections') %}

Sections

+
@@ -195,7 +248,7 @@

Sections

- +
NameTimePackageScalingFallbackApplicability
{{ item.get('time', 'N/A') }} {{ item.get('estimation_package', 'N/A') }} {{ item.get('scaling_method', 'N/A') }}{{ item.get('fallback_used', '—') }}{{ item.get('fallback_used', '—') }} {% if applicability %}
{{ applicability.get('status', 'N/A') }}
@@ -210,9 +263,11 @@

Sections

{% endfor %}
+
{% endif %} {% if future_breakdown.get('overlaps') %}

Overlaps

+
@@ -226,7 +281,7 @@

Overlaps

- +
SectionsTimePackageScalingFallbackApplicability
{{ item.get('time', 'N/A') }} {{ item.get('estimation_package', 'N/A') }} {{ item.get('scaling_method', 'N/A') }}{{ item.get('fallback_used', '—') }}{{ item.get('fallback_used', '—') }} {% if applicability %}
{{ applicability.get('status', 'N/A') }}
@@ -241,6 +296,7 @@

Overlaps

{% endfor %}
+
{% endif %} {% else %}
No section or overlap breakdown is stored for the future-side estimate.
From fba4efc455cead2e45bb85c2cb5c7f72d39c09b2 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 17:14:41 +0900 Subject: [PATCH 03/89] feat: improve result detail readability --- result_server/templates/result_detail.html | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html index 6e7b5e9..326599e 100644 --- a/result_server/templates/result_detail.html +++ b/result_server/templates/result_detail.html @@ -13,6 +13,44 @@ .meta-table th { text-align: left; min-width: 150px; background-color: #f2f2f2; } .meta-table td { min-width: 200px; } .quality-table th { min-width: 200px; width: 200px; } + .detail-table-wrap { overflow-x: auto; } + .meta-table th, + .quality-table th, + .scalar-table thead th, + .build-table th { + position: sticky; + top: 0; + z-index: 2; + box-shadow: inset 0 -1px 0 #d8e3e8; + } + .meta-table td, + .quality-table td, + .build-table td { + white-space: normal; + max-width: none; + vertical-align: top; + line-height: 1.45; + } + .meta-table td:first-child, + .meta-table th:first-child, + .quality-table td:first-child, + .quality-table th:first-child, + .scalar-table td:first-child, + .scalar-table th:first-child, + .build-table td:first-child, + .build-table th:first-child { + position: sticky; + left: 0; + z-index: 1; + background: #f9fcfd; + box-shadow: 10px 0 18px -18px rgba(15, 23, 42, 0.28); + } + .meta-table th:first-child, + .quality-table th:first-child, + .scalar-table th:first-child, + .build-table th:first-child { + z-index: 3; + } .chart-container { position: relative; width: 100%; max-width: 900px; height: 400px; margin: 0 auto; } .chart-fallback { padding: 20px; text-align: center; color: #999; border: 1px dashed #ddd; border-radius: 4px; } .scalar-table { width: auto; } @@ -40,6 +78,7 @@

Result Detail

Meta Information

+
@@ -57,11 +96,13 @@

Meta Information

{% endif %}
Code{{ result.get('code', 'N/A') }}
System{{ result.get('system', 'N/A') }}
CPUs per Node{{ result.get('cpus_per_node') }}
+
{% if quality %}

Quality

+
@@ -113,6 +154,7 @@

Quality

Level
+
{% endif %} @@ -135,6 +177,7 @@

Vector Metrics - Graph

Vector Metrics - Data Table

+
@@ -157,6 +200,7 @@

Vector Metrics - Data Table

{% endfor %}
+
{% endif %} @@ -167,6 +211,7 @@

Vector Metrics - Data Table

{% if scalar_keys | length >= 2 %}

Scalar Metrics

+
@@ -177,6 +222,7 @@

Scalar Metrics

{% endfor %}
MetricValue
+
{% endif %} {% endif %} @@ -186,6 +232,7 @@

Scalar Metrics

{% set build = result['build'] %}

Build Information

+
{% if build.get('spack') %} @@ -210,6 +257,7 @@

Build Information

{% endif %} {% endif %}
Build Tool{{ build.get('tool', 'N/A') }}
+
{% endif %} From dd29ff1eb0734d6025dc9c3e7dca194dfcbb551b Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 17:24:29 +0900 Subject: [PATCH 04/89] feat: improve system list usability --- result_server/app.py | 8 +- result_server/app_dev.py | 8 +- result_server/templates/home.html | 35 ++- result_server/templates/systemlist.html | 203 +++++++++++++++++- .../tests/test_systemlist_template.py | 96 +++++++++ result_server/utils/system_info.py | 17 ++ 6 files changed, 351 insertions(+), 16 deletions(-) create mode 100644 result_server/tests/test_systemlist_template.py diff --git a/result_server/app.py b/result_server/app.py index f4112b9..f3a1fff 100644 --- a/result_server/app.py +++ b/result_server/app.py @@ -108,9 +108,13 @@ def create_app(prefix="", base_dir=None): @app.route(f"{prefix}/systemlist") def systemlist(): - from utils.system_info import get_all_systems_info + from utils.system_info import get_all_systems_info, summarize_systems_info systems_info = get_all_systems_info() - return render_template("systemlist.html", systems_info=systems_info) + return render_template( + "systemlist.html", + systems_info=systems_info, + systems_summary=summarize_systems_info(systems_info), + ) return app diff --git a/result_server/app_dev.py b/result_server/app_dev.py index 386e665..d7c350c 100644 --- a/result_server/app_dev.py +++ b/result_server/app_dev.py @@ -114,7 +114,7 @@ def create_dev_app(base_dir): from flask_session import Session from routes.home import register_home_routes - from utils.system_info import get_all_systems_info + from utils.system_info import get_all_systems_info, summarize_systems_info app = Flask(__name__, template_folder="templates") @@ -167,7 +167,11 @@ def create_dev_app(base_dir): @app.route("/systemlist") def systemlist(): systems_info = get_all_systems_info() - return render_template("systemlist.html", systems_info=systems_info) + return render_template( + "systemlist.html", + systems_info=systems_info, + systems_summary=summarize_systems_info(systems_info), + ) return app diff --git a/result_server/templates/home.html b/result_server/templates/home.html index 38f1103..945ff11 100644 --- a/result_server/templates/home.html +++ b/result_server/templates/home.html @@ -208,10 +208,34 @@ .system-card { padding: 18px; } + .system-card-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + margin-bottom: 12px; + } .system-card h3 { - margin: 0 0 12px; + margin: 0; font-size: 18px; } + .system-badge { + display: inline-flex; + align-items: center; + padding: 4px 8px; + border-radius: 999px; + font-size: 12px; + font-weight: 600; + white-space: nowrap; + } + .system-badge.gpu { + background: #eef6ff; + color: #1d4ed8; + } + .system-badge.cpu { + background: #f3f4f6; + color: #4b5563; + } .system-meta { display: grid; gap: 7px; @@ -359,7 +383,14 @@

Available Systems

{% for system in systems %}
-

{{ system.name }}

+
+

{{ system.name }}

+ {% if system.gpu_name != '-' %} + GPU-enabled + {% else %} + CPU-only + {% endif %} +
CPU {{ system.cpu_name }}
CPU/node {{ system.cpu_per_node }} | Cores {{ system.cpu_cores }}
diff --git a/result_server/templates/systemlist.html b/result_server/templates/systemlist.html index 8a13dc2..cb51df2 100644 --- a/result_server/templates/systemlist.html +++ b/result_server/templates/systemlist.html @@ -4,6 +4,157 @@ Available Computing Systems {% include "_table_base.html" %} + + @@ -17,8 +168,37 @@

Available Systems

-
- +
+
+ {{ systems_summary.total_count }} + Connected systems registered in the portal. +
+
+ {{ systems_summary.gpu_enabled_count }} + Systems with GPU accelerators listed in system_info.csv. +
+
+ {{ systems_summary.cpu_only_count }} + CPU-only systems available for benchmark onboarding and browsing. +
+
+ +
+ + + + + {{ systems_summary.total_count }} visible +
+ +
+
@@ -32,19 +212,22 @@

Available Systems

{% for system_name, info in systems_info.items() %} - - - - - - - - + + + + + + + + {% endfor %}
SYSTEM
{{ info.name }}{{ info.cpu_name }}{{ info.cpu_per_node }}{{ info.cpu_cores }}{{ info.gpu_name }}{{ info.gpu_per_node }}{{ info.memory }}
{{ info.name }}{{ info.cpu_name }}{{ info.cpu_per_node }}{{ info.cpu_cores }}{{ info.gpu_name }}{{ info.gpu_per_node }}{{ info.memory }}
+

+ Hardware summaries are sourced from config/system_info.csv. Use this page for quick orientation, then follow the onboarding guides when adding new systems or applications. +

diff --git a/result_server/tests/test_systemlist_template.py b/result_server/tests/test_systemlist_template.py new file mode 100644 index 0000000..80ea052 --- /dev/null +++ b/result_server/tests/test_systemlist_template.py @@ -0,0 +1,96 @@ +import os +import sys +import types + + +def _setup_stubs(): + if "redis" not in sys.modules: + sys.modules["redis"] = types.ModuleType("redis") + + +_setup_stubs() + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from flask import Flask, Blueprint + + +def test_systemlist_page_renders_summary_and_table(): + app = Flask( + __name__, + template_folder=os.path.join(os.path.dirname(__file__), "..", "templates"), + ) + app.config["TESTING"] = True + app.config["SECRET_KEY"] = "test-secret" + + @app.route("/") + def home(): + return "" + + @app.route("/systemlist") + def systemlist(): + return "" + + results_bp = Blueprint("results", __name__) + estimated_bp = Blueprint("estimated", __name__) + auth_bp = Blueprint("auth", __name__) + + @results_bp.route("/") + def results(): + return "" + + @estimated_bp.route("/") + def estimated_results(): + return "" + + @auth_bp.route("/login") + def login(): + return "" + + app.register_blueprint(results_bp, url_prefix="/results") + app.register_blueprint(estimated_bp, url_prefix="/estimated") + app.register_blueprint(auth_bp, url_prefix="/auth") + + with app.test_request_context("/systemlist"): + from flask import render_template + + html = render_template( + "systemlist.html", + systems_summary={ + "total_count": 2, + "gpu_enabled_count": 1, + "cpu_only_count": 1, + }, + systems_info={ + "Fugaku": { + "name": "Fugaku", + "cpu_name": "A64FX", + "cpu_per_node": "1", + "cpu_cores": "48", + "gpu_name": "-", + "gpu_per_node": "-", + "memory": "32GB", + }, + "MiyabiG": { + "name": "MiyabiG", + "cpu_name": "NVIDIA Grace CPU", + "cpu_per_node": "1", + "cpu_cores": "72", + "gpu_name": "NVIDIA Hopper H100 GPU", + "gpu_per_node": "1", + "memory": "120GB", + }, + }, + ) + + assert "Available Systems" in html + assert "Connected systems registered in the portal." in html + assert "Systems with GPU accelerators listed in" in html + assert "systems-table" in html + assert "systems-table-wrap" in html + assert "Filter systems by name, CPU, GPU, or memory" in html + assert "GPU-enabled" in html + assert "CPU-only" in html + assert "Hardware summaries are sourced from" in html + assert "Fugaku" in html + assert "MiyabiG" in html diff --git a/result_server/utils/system_info.py b/result_server/utils/system_info.py index b7f404d..8fb69f6 100644 --- a/result_server/utils/system_info.py +++ b/result_server/utils/system_info.py @@ -76,3 +76,20 @@ def get_all_systems_info(): if name not in ordered: ordered[name] = info return ordered + + +def summarize_systems_info(systems_info): + """Return lightweight summary counts for the system list page.""" + total_count = len(systems_info) + gpu_enabled_count = 0 + + for info in systems_info.values(): + gpu_name = (info.get("gpu_name") or "-").strip() + if gpu_name and gpu_name != "-": + gpu_enabled_count += 1 + + return { + "total_count": total_count, + "gpu_enabled_count": gpu_enabled_count, + "cpu_only_count": total_count - gpu_enabled_count, + } From 93d3544f194597e181d312b2f1bb1e1dce4492ed Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 17:30:47 +0900 Subject: [PATCH 05/89] feat: polish portal entry and auth pages --- result_server/templates/admin_users.html | 63 +++++++--- result_server/templates/auth_login.html | 128 ++++++++++++++----- result_server/templates/auth_setup.html | 128 +++++++++++++++---- result_server/templates/home.html | 40 ++++-- result_server/templates/usage_report.html | 24 ++-- result_server/tests/test_auth_templates.py | 140 +++++++++++++++++++++ 6 files changed, 434 insertions(+), 89 deletions(-) create mode 100644 result_server/tests/test_auth_templates.py diff --git a/result_server/templates/admin_users.html b/result_server/templates/admin_users.html index ac8d7ee..712b713 100644 --- a/result_server/templates/admin_users.html +++ b/result_server/templates/admin_users.html @@ -25,7 +25,35 @@ .status-ok { color: #15803d; font-weight: 600; } .status-pending { color: #b91c1c; font-weight: 600; } .inline-form { display: inline; } - .actions { white-space: nowrap; } + .actions { min-width: 280px; } + .actions-stack { + display: flex; + flex-wrap: wrap; + gap: 8px; + } + .users-table thead th { + position: sticky; + top: 0; + z-index: 2; + box-shadow: inset 0 -1px 0 #d8e3e8; + } + .users-table td:first-child, + .users-table th:first-child { + position: sticky; + left: 0; + z-index: 1; + background: #f9fcfd; + box-shadow: 10px 0 18px -18px rgba(15, 23, 42, 0.28); + } + .users-table thead th:first-child { + z-index: 3; + background: #eef6f8; + } + .users-table td { + max-width: none; + white-space: normal; + vertical-align: top; + } @@ -67,8 +95,9 @@

Add User

Registered Users

+

Review current user access, TOTP registration status, and affiliation updates in one place.

- +
@@ -90,20 +119,22 @@

Registered Users

{% endif %} {% else %} diff --git a/result_server/templates/auth_login.html b/result_server/templates/auth_login.html index 4570625..699cce0 100644 --- a/result_server/templates/auth_login.html +++ b/result_server/templates/auth_login.html @@ -3,33 +3,94 @@ Login - CX Portal + {% include "_table_base.html" %} {% include "_navigation.html" %} - + + {% if session.get('authenticated') and 'admin' in session.get('user_affiliations', []) %} +
+

Operations Views

+

+ These pages help administrators check node-hour usage, application/system coverage, configuration state, and portal access setup. +

+
+ + Usage Report + Inspect node-hour usage together with coverage checks, latest result quality state, and source provenance state. + + + User Management + Manage user affiliations, invitation links, and TOTP onboarding for the portal. + +
+
+ {% endif %}
diff --git a/result_server/templates/usage_report.html b/result_server/templates/usage_report.html index 82293f8..a4db2d4 100644 --- a/result_server/templates/usage_report.html +++ b/result_server/templates/usage_report.html @@ -1,7 +1,7 @@ {% extends "_results_base.html" %} {% block title %}Usage Report{% endblock %} -{% block page_subtitle %}Review node-hour usage across systems, periods, and applications, and check application/system coverage from registered run settings.{% endblock %} +{% block page_subtitle %}Review node-hour usage, registered coverage, configuration checks, and latest result current-state across applications and systems.{% endblock %} {% block content %}
@@ -239,7 +239,7 @@

Configuration Checks

-

Lightweight diagnostics for registered systems, queue definitions, and application support coverage.

+

Lightweight checks for registered systems, queue definitions, and application support completeness.

Registered Systems

@@ -296,7 +296,7 @@

Partial Application Support

Application/System Coverage

-

Quick matrix of which registered systems are enabled in each application's list.csv.

+

Current support matrix based on list.csv plus detected build.sh / run.sh system branches.

★ enabled and implemented △ enabled in list.csv, script support incomplete @@ -340,7 +340,7 @@

Application/System Coverage

Result Quality Coverage

-

Current-state check based on the latest collected result for each application/system pair, showing whether top-level source tracking, breakdown output, and estimation-ready fields are present.

+

Current-state check based on the latest collected result for each application/system pair, showing source provenance, breakdown output, and estimation-ready fields.

{% if result_quality_rollup.rows %}
Email -
- - -
- {% if u.email != session.get('user_email') %} -
- -
-
- -
- {% endif %} +
+
+ + +
+ {% if u.email != session.get('user_email') %} +
+ +
+
+ +
+ {% endif %} +
@@ -348,14 +348,14 @@

Result Quality Coverage

- - - - - - - - + + + + + + + + diff --git a/result_server/tests/test_auth_templates.py b/result_server/tests/test_auth_templates.py new file mode 100644 index 0000000..9fceb0d --- /dev/null +++ b/result_server/tests/test_auth_templates.py @@ -0,0 +1,140 @@ +import os +import sys +import types + + +def _setup_stubs(): + if "redis" not in sys.modules: + sys.modules["redis"] = types.ModuleType("redis") + + +_setup_stubs() + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from flask import Flask, Blueprint + + +def _make_app(): + app = Flask( + __name__, + template_folder=os.path.join(os.path.dirname(__file__), "..", "templates"), + ) + app.config["TESTING"] = True + app.config["SECRET_KEY"] = "test-secret" + + results_bp = Blueprint("results", __name__) + estimated_bp = Blueprint("estimated", __name__) + auth_bp = Blueprint("auth", __name__) + admin_bp = Blueprint("admin", __name__) + + @results_bp.route("/") + def results(): + return "" + + @estimated_bp.route("/") + def estimated_results(): + return "" + + @auth_bp.route("/login") + def login(): + return "" + + @auth_bp.route("/setup/") + def setup(token): + return token + + @admin_bp.route("/users") + def users(): + return "" + + @admin_bp.route("/users/add", methods=["POST"]) + def add_user(): + return "" + + @admin_bp.route("/users//affiliations", methods=["POST"]) + def update_affiliations(email): + return email + + @admin_bp.route("/users//reinvite", methods=["POST"]) + def reinvite_user(email): + return email + + @admin_bp.route("/users//delete", methods=["POST"]) + def delete_user(email): + return email + + app.register_blueprint(results_bp, url_prefix="/results") + app.register_blueprint(estimated_bp, url_prefix="/estimated") + app.register_blueprint(auth_bp, url_prefix="/auth") + app.register_blueprint(admin_bp, url_prefix="/admin") + + @app.route("/") + def home(): + return "" + + @app.route("/systemlist") + def systemlist(): + return "" + + return app + + +def test_auth_login_template_renders_portal_shell(): + app = _make_app() + with app.test_request_context("/auth/login"): + from flask import render_template + + html = render_template("auth_login.html", step="email") + + assert "Sign in with your email address and TOTP code" in html + assert "Step 2 of 2" not in html + assert "Continue" in html + + +def test_auth_setup_template_renders_portal_shell(): + app = _make_app() + with app.test_request_context("/auth/setup/token-1"): + from flask import render_template + + html = render_template( + "auth_setup.html", + error=False, + qr_data="data:image/png;base64,abc", + secret="SECRETKEY", + email="user@example.com", + token="token-1", + ) + + assert "Complete portal access setup" in html + assert "Invitation-based setup" in html + assert "Complete Setup" in html + + +def test_admin_users_template_renders_portal_table(): + app = _make_app() + with app.test_request_context("/admin/users"): + from flask import render_template, session + + session["user_email"] = "admin@example.com" + html = render_template( + "admin_users.html", + users=[ + { + "email": "admin@example.com", + "affiliations": ["admin"], + "has_totp": True, + }, + { + "email": "user@example.com", + "affiliations": ["dev"], + "has_totp": False, + }, + ], + invitation_url=None, + ) + + assert "Manage access, affiliations, and TOTP onboarding" in html + assert "Review current user access" in html + assert "Registered" in html + assert "Pending" in html From ab101e566e543dfc76239b216e514cc450f25662 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 18:19:49 +0900 Subject: [PATCH 06/89] feat: improve portal list readability --- result_server/templates/_results_table.html | 59 ++++- .../templates/estimated_results.html | 68 ++++- result_server/templates/usage_report.html | 27 ++ .../tests/test_portal_list_templates.py | 233 ++++++++++++++++++ 4 files changed, 383 insertions(+), 4 deletions(-) create mode 100644 result_server/tests/test_portal_list_templates.py diff --git a/result_server/templates/_results_table.html b/result_server/templates/_results_table.html index b573e79..fbefac4 100644 --- a/result_server/templates/_results_table.html +++ b/result_server/templates/_results_table.html @@ -2,7 +2,63 @@ {% include "_pagination.html" %} -
Application SystemLatest ResultSource StatusSource TypeSource ReferenceMissing Source FieldsBreakdown PresentEstimation-ReadyRichLatest ResultTrackedTypeReferenceMissing FieldsBreakdownEst.-ReadyRich Warnings
+ + +

Use the server-side filters to narrow the table first, then select visible rows if you want to open the compare view.

+ +
+
@@ -191,6 +247,7 @@ {% endfor %}
+
+ +

Focus on system pairs, applied packages, and ratio at the list level, then open detail when you need breakdowns or fallback/applicability context.

+ +
+ @@ -137,7 +199,7 @@ -
Timestamp{{ applied_short }}{% if r.method_class or r.detail_level %}
{{ r.method_class }}{% if r.detail_level %} / {{ r.detail_level }}{% endif %}
{% endif %}		 {{ r.estimate_uuid[:8] if r.estimate_uuid else '' }} {{ ratio }} + detail | json diff --git a/result_server/templates/usage_report.html b/result_server/templates/usage_report.html index a4db2d4..af8953e 100644 --- a/result_server/templates/usage_report.html +++ b/result_server/templates/usage_report.html @@ -137,6 +137,10 @@ color: #0f766e; font-weight: 600; } + .usage-search-box { + min-width: 280px; + flex: 1 1 320px; + } @@ -170,6 +174,14 @@ {% endfor %} {% endif %} + + {% if result.apps %} @@ -382,4 +394,19 @@

Result Quality Coverage

No collected result-quality data is available yet.

{% endif %} + + {% endblock %} diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py new file mode 100644 index 0000000..d35a457 --- /dev/null +++ b/result_server/tests/test_portal_list_templates.py @@ -0,0 +1,233 @@ +import os +import sys +import types + + +def _setup_stubs(): + if "redis" not in sys.modules: + sys.modules["redis"] = types.ModuleType("redis") + + +_setup_stubs() + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from flask import Flask, Blueprint + + +def _make_app(): + app = Flask( + __name__, + template_folder=os.path.join(os.path.dirname(__file__), "..", "templates"), + ) + app.config["TESTING"] = True + app.config["SECRET_KEY"] = "test-secret" + + results_bp = Blueprint("results", __name__) + estimated_bp = Blueprint("estimated", __name__) + auth_bp = Blueprint("auth", __name__) + admin_bp = Blueprint("admin", __name__) + + @results_bp.route("/") + def results(): + return "" + + @results_bp.route("/compare") + def result_compare(): + return "" + + @results_bp.route("/detail/") + def result_detail(filename): + return filename + + @results_bp.route("/usage") + def usage_report(): + return "" + + @estimated_bp.route("/") + def estimated_results(): + return "" + + @estimated_bp.route("/detail/") + def estimated_detail(filename): + return filename + + @estimated_bp.route("/show/") + def show_estimated_result(filename): + return filename + + @auth_bp.route("/login") + def login(): + return "" + + @admin_bp.route("/users") + def users(): + return "" + + app.register_blueprint(results_bp, url_prefix="/results") + app.register_blueprint(estimated_bp, url_prefix="/estimated") + app.register_blueprint(auth_bp, url_prefix="/auth") + app.register_blueprint(admin_bp, url_prefix="/admin") + + @app.route("/") + def home(): + return "" + + @app.route("/systemlist") + def systemlist(): + return "" + + return app + + +def test_results_template_renders_table_note(): + app = _make_app() + with app.test_request_context("/results"): + from flask import render_template + + html = render_template( + "results.html", + columns=[ + ("Timestamp", "timestamp"), + ("SYSTEM", "system"), + ("CODE", "code"), + ("FOM", "fom"), + ("Exp", "exp"), + ("JSON", "json_link"), + ], + rows=[ + { + "timestamp": "2026-04-13 12:00:00", + "system": "Fugaku", + "code": "qws", + "fom": 1.234, + "exp": "CASE0", + "json_link": "/results/result0.json", + "data_link": None, + "filename": "result0.json", + "detail_link": "/results/detail/result0.json", + "source_info": None, + "quality": { + "level": "ready", + "label": "Ready", + "summary": "Breakdown is present.", + }, + "fom_version": "DDSolverJacobi", + "nodes": "1", + "numproc_node": "1", + "nthreads": "12", + "ci_trigger": "push", + "pipeline_id": "10", + "source_hash": "main@abcdef12", + } + ], + pagination={"total": 1, "page": 1, "total_pages": 1}, + current_per_page=50, + current_system="", + current_code="", + current_exp="", + filter_options={"systems": ["Fugaku"], "codes": ["qws"], "exps": ["CASE0"]}, + systems_info={ + "Fugaku": { + "name": "Fugaku", + "cpu_name": "A64FX", + "cpu_per_node": "1", + "cpu_cores": "48", + "gpu_name": "-", + "gpu_per_node": "-", + "memory": "32GB", + } + }, + ) + + assert "Use the server-side filters to narrow the table first" in html + assert "results-table-wrap" in html + assert "Compare" in html + + +def test_estimated_results_template_renders_table_note(): + app = _make_app() + with app.test_request_context("/estimated"): + from flask import render_template + + html = render_template( + "estimated_results.html", + authenticated=True, + rows=[ + { + "timestamp": "2026-04-13 12:00:00", + "code": "qws", + "exp": "CASE0", + "systemA_system": "Fugaku", + "systemA_fom": 0.944, + "systemA_target_nodes": "1024", + "systemA_scaling_method": "weakscaling", + "systemA_bench_system": "Fugaku", + "systemA_bench_fom": 0.386, + "systemA_bench_nodes": "1", + "systemB_system": "FugakuNEXT", + "systemB_fom": 9.054, + "systemB_target_nodes": "256", + "systemB_scaling_method": "instrumented-app-sections-dummy", + "systemB_bench_system": "MiyabiG", + "systemB_bench_fom": 5.712, + "systemB_bench_nodes": "1", + "applicability_status": "applicable", + "requested_estimation_package": "instrumented_app_sections_dummy", + "estimation_package": "instrumented_app_sections_dummy", + "requested_current_estimation_package": "weakscaling", + "requested_future_estimation_package": "instrumented_app_sections_dummy", + "current_estimation_package": "weakscaling", + "future_estimation_package": "instrumented_app_sections_dummy", + "method_class": "detailed", + "detail_level": "intermediate", + "estimate_uuid": "11111111-2222-3333-4444-555555555555", + "performance_ratio": 0.104, + "json_link": "estimate0.json", + } + ], + pagination={"total": 1, "page": 1, "total_pages": 1}, + current_per_page=50, + current_system="", + current_code="", + current_exp="", + filter_options={"systems": ["Fugaku"], "codes": ["qws"], "exps": ["CASE0"]}, + ) + + assert "Focus on system pairs, applied packages, and ratio" in html + assert "estimated-table-wrap" in html + assert "detail" in html + + +def test_usage_report_template_renders_search_box(): + app = _make_app() + with app.test_request_context("/results/usage"): + from flask import render_template + + html = render_template( + "usage_report.html", + result={ + "apps": [], + "systems": [], + "periods": [], + "available_fiscal_years": [2025], + }, + filtered_periods=[], + period_type="fiscal_year", + fiscal_year=2025, + period_filter="", + site_diagnostics={ + "registered_system_count": 1, + "unused_systems": [], + "missing_system_info": [], + "missing_queue_definitions": [], + "application_count": 0, + "partial_support": [], + }, + coverage_systems=[], + app_support_rows=[], + result_quality_rollup={"rows": []}, + ) + + assert "Filter application/system coverage and current-state tables" in html + assert "applyUsageSearch" in html From f51a3e672effc7ed283450768677e765034d0c80 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 18:28:04 +0900 Subject: [PATCH 07/89] fix: simplify portal list table behavior --- result_server/templates/_results_table.html | 37 +------------------ .../templates/estimated_results.html | 37 +------------------ 2 files changed, 2 insertions(+), 72 deletions(-) diff --git a/result_server/templates/_results_table.html b/result_server/templates/_results_table.html index fbefac4..ceaefeb 100644 --- a/result_server/templates/_results_table.html +++ b/result_server/templates/_results_table.html @@ -5,48 +5,13 @@ -

Use the server-side filters to narrow the table first, then select visible rows if you want to open the compare view.

+

Use the server-side filters to narrow the table first, check the profiler summary when PA data is available, and then select visible rows if you want to open the compare view.

@@ -28,7 +28,7 @@ {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "JSON", "PA Data", "Trigger", "Pipeline", "Branch/Hash"] %} + {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler", "JSON", "PA Data", "Trigger", "Pipeline", "Branch/Hash"] %} {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "JSON", "PA Data", "Trigger", "Pipeline", "Branch/Hash"] %} + {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler", "JSON", "PA Data", "Trigger", "Pipeline", "Branch/Hash"] %} {% if col_name == "SYSTEM" %} {% elif key == "timestamp" %} {% set ts_parts = row[key].split(' ') %} @@ -251,6 +274,22 @@ background: #e8f1ff; color: #1f4f99; } +.profile-cell { + min-width: 150px; + white-space: normal; +} +.profile-headline { + display: block; + font-weight: 600; + line-height: 1.35; +} +.profile-subline { + display: block; + margin-top: 0.15rem; + color: #607282; + font-size: 0.82em; + line-height: 1.35; +} {% include "_pagination.html" %} diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html index 8259300..8561c00 100644 --- a/result_server/templates/result_detail.html +++ b/result_server/templates/result_detail.html @@ -108,6 +108,27 @@

PA Data Summary

+ + + + - + diff --git a/result_server/tests/test_result_detail_template.py b/result_server/tests/test_result_detail_template.py index c5633b3..87c56dd 100644 --- a/result_server/tests/test_result_detail_template.py +++ b/result_server/tests/test_result_detail_template.py @@ -164,7 +164,8 @@ def test_pa_data_summary_section(self, app): assert "PA Data Summary" in html assert "fapp" in html assert "single" in html - assert "single preset: pa1" in html + assert "Tool-Specific Events" in html + assert "fapp event set: pa1" in html assert "summary_text" in html assert "pa1" in html From 670610882ee778fd77180486649640c7d4e09626 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 23:39:20 +0900 Subject: [PATCH 18/89] chore: skip ci for profiler wording cleanup [skip-ci] From 01b3b528bdb96d05bfadbc6f1cdabee3193ba86d Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 00:02:54 +0900 Subject: [PATCH 19/89] feat: switch qws Fugaku profiling to simple [code:qws] [system:Fugaku] --- programs/qws/run.sh | 2 +- scripts/tests/test_bk_profiler.sh | 56 +++++++++++++++---------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/programs/qws/run.sh b/programs/qws/run.sh index 44af66e..2e7614b 100644 --- a/programs/qws/run.sh +++ b/programs/qws/run.sh @@ -8,7 +8,7 @@ export OMP_NUM_THREADS=$nthreads source "${PWD}/scripts/bk_functions.sh" qws_profiler_tool="fapp" -qws_profiler_level="single" +qws_profiler_level="simple" # Load estimation helpers used when emitting section/overlap metadata. source "${PWD}/programs/qws/estimate.sh" diff --git a/scripts/tests/test_bk_profiler.sh b/scripts/tests/test_bk_profiler.sh index c131719..c0835d7 100644 --- a/scripts/tests/test_bk_profiler.sh +++ b/scripts/tests/test_bk_profiler.sh @@ -77,41 +77,39 @@ EOF chmod +x "${FAKE_BIN}/fapp" "${FAKE_BIN}/fapppx" export PATH="${FAKE_BIN}:${PATH}" -run_and_check_default() { - local archive="${TMP_DIR}/default.tgz" - local extract_dir="${TMP_DIR}/default_extract" - - bk_profiler fapp --archive "$archive" --raw-dir "${TMP_DIR}/default_pa" -- true +run_and_check_level() { + local level="$1" + local expected_last_rep="$2" + local expected_last_event="$3" + local expected_format="$4" + local expect_csv="$5" + local archive="${TMP_DIR}/${level}.tgz" + local extract_dir="${TMP_DIR}/${level}_extract" + local raw_dir="${TMP_DIR}/${level}_pa" + + bk_profiler fapp --level "$level" --archive "$archive" --raw-dir "$raw_dir" -- true mkdir -p "$extract_dir" tar -xzf "$archive" -C "$extract_dir" test -f "${extract_dir}/bk_profiler_artifact/meta.json" test -f "${extract_dir}/bk_profiler_artifact/raw/rep1/event.txt" - test -f "${extract_dir}/bk_profiler_artifact/reports/fapp_A_rep1.txt" - ! test -f "${extract_dir}/bk_profiler_artifact/reports/cpu_pa_rep1.csv" - grep -q '"level": "single"' "${extract_dir}/bk_profiler_artifact/meta.json" - grep -q '"report_format": "text"' "${extract_dir}/bk_profiler_artifact/meta.json" - grep -q '"event": "pa1"' "${extract_dir}/bk_profiler_artifact/meta.json" -} - -run_and_check_simple_both() { - local archive="${TMP_DIR}/simple.tgz" - local extract_dir="${TMP_DIR}/simple_extract" - - bk_profiler fapp --level simple --report-format both --archive "$archive" --raw-dir "${TMP_DIR}/simple_pa" -- true - mkdir -p "$extract_dir" - tar -xzf "$archive" -C "$extract_dir" - - test -f "${extract_dir}/bk_profiler_artifact/meta.json" - test -f "${extract_dir}/bk_profiler_artifact/raw/rep5/event.txt" - test -f "${extract_dir}/bk_profiler_artifact/reports/fapp_A_rep5.txt" - test -f "${extract_dir}/bk_profiler_artifact/reports/cpu_pa_rep5.csv" - grep -q '"level": "simple"' "${extract_dir}/bk_profiler_artifact/meta.json" - grep -q '"report_format": "both"' "${extract_dir}/bk_profiler_artifact/meta.json" - grep -q '"event": "pa5"' "${extract_dir}/bk_profiler_artifact/meta.json" + test -f "${extract_dir}/bk_profiler_artifact/raw/rep${expected_last_rep}/event.txt" + test -f "${extract_dir}/bk_profiler_artifact/reports/fapp_A_rep${expected_last_rep}.txt" + + if [ "$expect_csv" = "yes" ]; then + test -f "${extract_dir}/bk_profiler_artifact/reports/cpu_pa_rep${expected_last_rep}.csv" + else + ! test -f "${extract_dir}/bk_profiler_artifact/reports/cpu_pa_rep${expected_last_rep}.csv" + fi + + grep -q "\"level\": \"${level}\"" "${extract_dir}/bk_profiler_artifact/meta.json" + grep -q "\"report_format\": \"${expected_format}\"" "${extract_dir}/bk_profiler_artifact/meta.json" + grep -q "\"event\": \"${expected_last_event}\"" "${extract_dir}/bk_profiler_artifact/meta.json" } -run_and_check_default -run_and_check_simple_both +run_and_check_level single 1 pa1 text no +run_and_check_level simple 5 pa5 both yes +run_and_check_level standard 11 pa11 both yes +run_and_check_level detailed 17 pa17 both yes echo "bk_profiler tests passed" From 9ea613f4da97e5f55869ef7f0301aa8e151058a6 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 00:15:06 +0900 Subject: [PATCH 20/89] chore: log profiler repetitions for Fugaku debugging [code:qws] [system:Fugaku] --- scripts/bk_functions.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/bk_functions.sh b/scripts/bk_functions.sh index 9156b77..bc8ea6f 100644 --- a/scripts/bk_functions.sh +++ b/scripts/bk_functions.sh @@ -810,6 +810,15 @@ bk_profiler_write_meta() { } > "$_bk_meta_file" } +bk_profiler_call_optional_hook() { + _bk_hook_name="$1" + shift || true + + if declare -F "$_bk_hook_name" >/dev/null 2>&1; then + "$_bk_hook_name" "$@" + fi +} + bk_profiler() { if [ $# -lt 2 ]; then echo "bk_profiler: requires a profiler tool and an execution command" >&2 @@ -901,8 +910,12 @@ bk_profiler() { _bk_fapp_rep_name="rep${_bk_fapp_run_index}" _bk_fapp_rep_dir="${_bk_profiler_dir}/${_bk_fapp_rep_name}" mkdir -p "$_bk_fapp_rep_dir" + echo "bk_profiler[fapp]: starting ${_bk_fapp_rep_name} event=${_bk_fapp_event}" >&2 + bk_profiler_call_optional_hook bk_profiler_before_run "$_bk_profiler_tool" "$_bk_profiler_level" "$_bk_fapp_rep_name" "$_bk_fapp_event" "$@" || return 1 # shellcheck disable=SC2086 fapp -C -d "$_bk_fapp_rep_dir" ${BK_PROFILER_ARGS:-} -Hevent="${_bk_fapp_event}" "$@" + bk_profiler_call_optional_hook bk_profiler_after_run "$_bk_profiler_tool" "$_bk_profiler_level" "$_bk_fapp_rep_name" "$_bk_fapp_event" "$@" || return 1 + echo "bk_profiler[fapp]: completed ${_bk_fapp_rep_name} event=${_bk_fapp_event}" >&2 cp -R "$_bk_fapp_rep_dir" "$_bk_stage_dir/raw/${_bk_fapp_rep_name}" if [ -n "$_bk_profiler_run_names" ]; then _bk_profiler_run_names="${_bk_profiler_run_names},${_bk_fapp_rep_name}" From b40ecf3cdb4afeb80de769686feecc0c56dc8f3c Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 00:19:39 +0900 Subject: [PATCH 21/89] fix: follow latest Fugaku stdout path for qws profiler runs [code:qws] [system:Fugaku] --- programs/qws/run.sh | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/programs/qws/run.sh b/programs/qws/run.sh index 2e7614b..e6415e4 100644 --- a/programs/qws/run.sh +++ b/programs/qws/run.sh @@ -26,6 +26,26 @@ print_results() { qws_emit_estimation_data_from_fom "$fom" } +resolve_latest_fugaku_stdout() { + local stdout_name=$1 + local output_root="output.${PJM_JOBID}/0" + local candidate + local latest="" + + shopt -s nullglob + for candidate in "${output_root}"/*/"${stdout_name}"; do + latest="$candidate" + done + shopt -u nullglob + + if [[ -z "$latest" ]]; then + echo "ERROR: could not resolve Fugaku stdout path for ${stdout_name} under ${output_root}" >&2 + return 1 + fi + + printf '%s\n' "$latest" +} + emit_qws_dummy_padata() { mkdir -p pa echo dummy > ./pa/padat.0 @@ -51,16 +71,16 @@ case "$system" in case "$nodes" in 1) bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 - print_results output.${PJM_JOBID}/0/1/stdout.1.0 CASE0 1 >> ../results/result + print_results "$(resolve_latest_fugaku_stdout stdout.1.0)" CASE0 1 >> ../results/result if ! bk_profiler_enabled "$qws_profiler_tool"; then emit_qws_dummy_padata ../results/padata0.tgz fi mpiexec -n 2 ./main 32 6 4 3 1 1 1 2 -1 -1 6 50 > CASE1 - print_results output.${PJM_JOBID}/0/2/stdout.2.0 CASE1 2 >> ../results/result + print_results "$(resolve_latest_fugaku_stdout stdout.2.0)" CASE1 2 >> ../results/result ;; 2) bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 8 ./main 32 6 4 3 1 2 2 2 -1 -1 6 50 > CASE7 - print_results output.${PJM_JOBID}/0/1/stdout.1.0 CASE7 4 >> ../results/result + print_results "$(resolve_latest_fugaku_stdout stdout.1.0)" CASE7 4 >> ../results/result if ! bk_profiler_enabled "$qws_profiler_tool"; then emit_qws_dummy_padata ../results/padata0.tgz fi From 59ec1abd247f2aca34c9cc7ae38bcef2d2f7063a Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 00:22:47 +0900 Subject: [PATCH 22/89] fix: separate qws Fugaku profiling from FOM runs [code:qws] [system:Fugaku] --- programs/qws/run.sh | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/programs/qws/run.sh b/programs/qws/run.sh index e6415e4..2b074b5 100644 --- a/programs/qws/run.sh +++ b/programs/qws/run.sh @@ -26,26 +26,6 @@ print_results() { qws_emit_estimation_data_from_fom "$fom" } -resolve_latest_fugaku_stdout() { - local stdout_name=$1 - local output_root="output.${PJM_JOBID}/0" - local candidate - local latest="" - - shopt -s nullglob - for candidate in "${output_root}"/*/"${stdout_name}"; do - latest="$candidate" - done - shopt -u nullglob - - if [[ -z "$latest" ]]; then - echo "ERROR: could not resolve Fugaku stdout path for ${stdout_name} under ${output_root}" >&2 - return 1 - fi - - printf '%s\n' "$latest" -} - emit_qws_dummy_padata() { mkdir -p pa echo dummy > ./pa/padat.0 @@ -70,18 +50,22 @@ case "$system" in Fugaku|FugakuCN) case "$nodes" in 1) - bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 - print_results "$(resolve_latest_fugaku_stdout stdout.1.0)" CASE0 1 >> ../results/result - if ! bk_profiler_enabled "$qws_profiler_tool"; then + mpiexec -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0 + print_results output.${PJM_JOBID}/0/1/stdout.1.0 CASE0 1 >> ../results/result + mpiexec -n 2 ./main 32 6 4 3 1 1 1 2 -1 -1 6 50 > CASE1 + print_results output.${PJM_JOBID}/0/2/stdout.2.0 CASE1 2 >> ../results/result + if bk_profiler_enabled "$qws_profiler_tool"; then + bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 1 ./main 32 6 4 3 1 1 1 1 -1 -1 6 50 > CASE0.profile + else emit_qws_dummy_padata ../results/padata0.tgz fi - mpiexec -n 2 ./main 32 6 4 3 1 1 1 2 -1 -1 6 50 > CASE1 - print_results "$(resolve_latest_fugaku_stdout stdout.2.0)" CASE1 2 >> ../results/result ;; 2) - bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 8 ./main 32 6 4 3 1 2 2 2 -1 -1 6 50 > CASE7 - print_results "$(resolve_latest_fugaku_stdout stdout.1.0)" CASE7 4 >> ../results/result - if ! bk_profiler_enabled "$qws_profiler_tool"; then + mpiexec -n 8 ./main 32 6 4 3 1 2 2 2 -1 -1 6 50 > CASE7 + print_results output.${PJM_JOBID}/0/1/stdout.1.0 CASE7 4 >> ../results/result + if bk_profiler_enabled "$qws_profiler_tool"; then + bk_profiler "$qws_profiler_tool" --level "$qws_profiler_level" --archive ../results/padata0.tgz --raw-dir pa -- mpiexec -n 8 ./main 32 6 4 3 1 2 2 2 -1 -1 6 50 > CASE7.profile + else emit_qws_dummy_padata ../results/padata0.tgz fi ;; From dcb1972b0074b113da388603fbe7da78d21ad3e5 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 08:16:01 +0900 Subject: [PATCH 23/89] feat: switch qws Fugaku profiling to standard [code:qws] [system:Fugaku] --- programs/qws/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/qws/run.sh b/programs/qws/run.sh index 2b074b5..2e43ba5 100644 --- a/programs/qws/run.sh +++ b/programs/qws/run.sh @@ -8,7 +8,7 @@ export OMP_NUM_THREADS=$nthreads source "${PWD}/scripts/bk_functions.sh" qws_profiler_tool="fapp" -qws_profiler_level="simple" +qws_profiler_level="standard" # Load estimation helpers used when emitting section/overlap metadata. source "${PWD}/programs/qws/estimate.sh" From 6a53d7457bcce051ea4efcc04a51cc12b252e002 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 08:27:21 +0900 Subject: [PATCH 24/89] feat: switch qws Fugaku profiling to detailed [code:qws] [system:Fugaku] --- programs/qws/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/qws/run.sh b/programs/qws/run.sh index 2e43ba5..396c46a 100644 --- a/programs/qws/run.sh +++ b/programs/qws/run.sh @@ -8,7 +8,7 @@ export OMP_NUM_THREADS=$nthreads source "${PWD}/scripts/bk_functions.sh" qws_profiler_tool="fapp" -qws_profiler_level="standard" +qws_profiler_level="detailed" # Load estimation helpers used when emitting section/overlap metadata. source "${PWD}/programs/qws/estimate.sh" From bf0fa08f74f5db3f2e26092b84413d831ff51360 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 09:32:28 +0900 Subject: [PATCH 25/89] fix: keep portal tooltips visible --- result_server/templates/_table_base.html | 124 +++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/result_server/templates/_table_base.html b/result_server/templates/_table_base.html index d3e2e05..dca4b11 100644 --- a/result_server/templates/_table_base.html +++ b/result_server/templates/_table_base.html @@ -178,6 +178,30 @@ opacity: 1; } + body.tooltip-floating-enabled .tooltip .tooltiptext { + display: none !important; + } + + .tooltip-floating-layer { + position: fixed; + z-index: 10000; + max-width: min(320px, calc(100vw - 24px)); + padding: 10px; + border-radius: 6px; + background: #333; + color: #fff; + font-size: 12px; + line-height: 1.4; + box-shadow: 0 10px 24px rgba(15, 23, 42, 0.22); + pointer-events: none; + opacity: 0; + transition: opacity 120ms ease; + } + + .tooltip-floating-layer.is-visible { + opacity: 1; + } + /* 左端の列用ツールチップ（右寄せ） */ .tooltip-left .tooltiptext { left: 0; @@ -235,4 +259,104 @@ row.style.display = match ? "" : "none"; }); } + +(function() { + var floatingTooltip = null; + var activeTooltip = null; + var margin = 12; + + function ensureFloatingTooltip() { + if (floatingTooltip) return floatingTooltip; + floatingTooltip = document.createElement('div'); + floatingTooltip.className = 'tooltip-floating-layer'; + document.body.appendChild(floatingTooltip); + document.body.classList.add('tooltip-floating-enabled'); + return floatingTooltip; + } + + function getTooltipTextElement(target) { + return target ? target.querySelector('.tooltiptext') : null; + } + + function positionFloatingTooltip(target, layer) { + var rect = target.getBoundingClientRect(); + var layerRect = layer.getBoundingClientRect(); + var top = rect.top - layerRect.height - 10; + var left; + + if (target.classList.contains('tooltip-left')) { + left = rect.left; + } else if (target.classList.contains('tooltip-right')) { + left = rect.right - layerRect.width; + } else { + left = rect.left + (rect.width / 2) - (layerRect.width / 2); + } + + left = Math.max(margin, Math.min(left, window.innerWidth - layerRect.width - margin)); + if (top < margin) { + top = rect.bottom + 10; + } + top = Math.max(margin, Math.min(top, window.innerHeight - layerRect.height - margin)); + + layer.style.left = left + 'px'; + layer.style.top = top + 'px'; + } + + function showFloatingTooltip(target) { + var textEl = getTooltipTextElement(target); + if (!textEl) return; + + var layer = ensureFloatingTooltip(); + activeTooltip = target; + layer.innerHTML = textEl.innerHTML; + layer.classList.add('is-visible'); + positionFloatingTooltip(target, layer); + } + + function hideFloatingTooltip(target) { + if (!floatingTooltip) return; + if (target && activeTooltip !== target) return; + activeTooltip = null; + floatingTooltip.classList.remove('is-visible'); + floatingTooltip.innerHTML = ''; + } + + document.addEventListener('mouseover', function(event) { + var target = event.target.closest('.tooltip'); + if (!target) return; + showFloatingTooltip(target); + }); + + document.addEventListener('mouseout', function(event) { + var target = event.target.closest('.tooltip'); + if (!target) return; + var related = event.relatedTarget && event.relatedTarget.closest ? event.relatedTarget.closest('.tooltip') : null; + if (related === target) return; + hideFloatingTooltip(target); + }); + + document.addEventListener('focusin', function(event) { + var target = event.target.closest('.tooltip'); + if (!target) return; + showFloatingTooltip(target); + }); + + document.addEventListener('focusout', function(event) { + var target = event.target.closest('.tooltip'); + if (!target) return; + hideFloatingTooltip(target); + }); + + window.addEventListener('scroll', function() { + if (activeTooltip && floatingTooltip && floatingTooltip.classList.contains('is-visible')) { + positionFloatingTooltip(activeTooltip, floatingTooltip); + } + }, true); + + window.addEventListener('resize', function() { + if (activeTooltip && floatingTooltip && floatingTooltip.classList.contains('is-visible')) { + positionFloatingTooltip(activeTooltip, floatingTooltip); + } + }); +})(); From ddf8dc4222ea9589d8bdbe4425052ff3b3dc0a6f Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 09:36:25 +0900 Subject: [PATCH 26/89] fix: compact portal page headers --- result_server/templates/_table_base.html | 36 +++++++++++++++--------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/result_server/templates/_table_base.html b/result_server/templates/_table_base.html index dca4b11..a00ea26 100644 --- a/result_server/templates/_table_base.html +++ b/result_server/templates/_table_base.html @@ -18,40 +18,40 @@ .portal-page { max-width: 1240px; margin: 0 auto; - padding: 0 24px 40px; + padding: 0 24px 32px; } .page-hero { - margin: 20px 0 18px; - padding: 24px 28px; - border-radius: 18px; + margin: 12px 0 14px; + padding: 14px 18px; + border-radius: 16px; border: 1px solid #d8e3e8; background: rgba(255, 255, 255, 0.92); - box-shadow: 0 14px 36px rgba(18, 52, 77, 0.08); + box-shadow: 0 10px 24px rgba(18, 52, 77, 0.06); } .page-eyebrow { display: inline-block; - margin-bottom: 10px; - padding: 5px 10px; + margin-bottom: 6px; + padding: 4px 8px; border-radius: 999px; background: #dff7f2; color: #0f766e; - font-size: 12px; + font-size: 11px; font-weight: 700; letter-spacing: 0.08em; text-transform: uppercase; } .page-title { - margin: 0 0 8px; - font-size: clamp(28px, 4vw, 40px); - line-height: 1.08; + margin: 0 0 4px; + font-size: clamp(22px, 3.1vw, 30px); + line-height: 1.12; letter-spacing: -0.03em; } .page-subtitle { margin: 0; max-width: 72ch; color: #52606d; - line-height: 1.65; - font-size: 15px; + line-height: 1.5; + font-size: 14px; } .page-card { margin: 0 0 18px; @@ -240,10 +240,18 @@ .portal-page { padding: 0 14px 28px; } - .page-hero, .page-card { padding: 16px; } + .page-hero { + padding: 12px 14px; + } + .page-title { + font-size: 22px; + } + .page-subtitle { + font-size: 13px; + } } From a665e4db475facc6d849f2b395e13532f3242326 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 10:38:55 +0900 Subject: [PATCH 27/89] feat: simplify result list profiler access --- result_server/templates/_results_base.html | 2 +- result_server/templates/_results_table.html | 49 +++++++++++++------ .../templates/estimated_results.html | 2 +- .../tests/test_portal_list_templates.py | 9 ++-- result_server/tests/test_results_loader.py | 3 +- result_server/utils/results_loader.py | 3 +- 6 files changed, 42 insertions(+), 26 deletions(-) diff --git a/result_server/templates/_results_base.html b/result_server/templates/_results_base.html index c1bcf32..ee86718 100644 --- a/result_server/templates/_results_base.html +++ b/result_server/templates/_results_base.html @@ -13,7 +13,7 @@
CX Portal

{% block page_title %}{{ self.title() }}{% endblock %}

-

{% block page_subtitle %}Browse portal data, compare systems, and inspect result details collected from connected environments.{% endblock %}

+

{% block page_subtitle %}Browse results, compare runs, and open detail pages from connected environments.{% endblock %}

{% block auth_warning %}{% endblock %} diff --git a/result_server/templates/_results_table.html b/result_server/templates/_results_table.html index d2a44e2..984e603 100644 --- a/result_server/templates/_results_table.html +++ b/result_server/templates/_results_table.html @@ -20,7 +20,7 @@ } -

Use the server-side filters to narrow the table first, check the profiler summary when PA data is available, and then select visible rows if you want to open the compare view.

+

Filter first, check the profiler and PA summary when available, and then select visible rows if you want to open compare.

{% if col_name == "Timestamp" %} {{ col_name }} @@ -66,6 +66,10 @@ {{ col_name }} Performance Analysis data archive (detailed profiling data) + {% elif col_name == "Profiler" %} + {{ col_name }} + Profiler tool, level, and report summary extracted from uploaded PA metadata + {% elif col_name == "Trigger" %} {{ col_name }} CI pipeline trigger source (schedule, push, web, etc.) @@ -102,7 +106,7 @@
+ {% if row.profile_summary_meta.has_profile_data %} + + {{ row.profile_summary_meta.headline }} + {% if row.profile_summary_meta.subline %} + {{ row.profile_summary_meta.subline }} + {% endif %} + + {{ row.profile_summary_meta.headline }} + {% if row.profile_summary_meta.subline %}
{{ row.profile_summary_meta.subline }}{% endif %} + {% if row.profile_summary_meta.events %}
events: {{ row.profile_summary_meta.events | join(', ') }}{% endif %} + {% if row.profile_summary_meta.report_kinds %}
reports: {{ row.profile_summary_meta.report_kinds | join(', ') }}{% endif %} + + + {% else %} + - + {% endif %} + 		{{ ts_parts[0] }}
{{ ts_parts[1] if ts_parts|length > 1 else '' }}
Level{{ result['profile_data'].get('level', 'N/A') }}
Report Format{{ result['profile_data'].get('report_format', 'N/A') }}
Run Count{{ result['profile_data'].get('run_count', 'N/A') }}
Level Summary + {% set level = result['profile_data'].get('level') %} + {% if result['profile_data'].get('tool') == 'fapp' %} + {% if level == 'single' %} + single preset: pa1 + {% elif level == 'simple' %} + simple preset: pa1..pa5 + {% elif level == 'standard' %} + standard preset: pa1..pa11 + {% elif level == 'detailed' %} + detailed preset: pa1..pa17 + {% else %} + tool-specific preset + {% endif %} + {% else %} + tool-specific preset + {% endif %} +
Events diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index d35a457..799f1d5 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -87,14 +87,15 @@ def test_results_template_renders_table_note(): html = render_template( "results.html", - columns=[ - ("Timestamp", "timestamp"), - ("SYSTEM", "system"), - ("CODE", "code"), - ("FOM", "fom"), - ("Exp", "exp"), - ("JSON", "json_link"), - ], + columns=[ + ("Timestamp", "timestamp"), + ("SYSTEM", "system"), + ("CODE", "code"), + ("FOM", "fom"), + ("Exp", "exp"), + ("Profiler", "profile_summary"), + ("JSON", "json_link"), + ], rows=[ { "timestamp": "2026-04-13 12:00:00", @@ -119,6 +120,14 @@ def test_results_template_renders_table_note(): "ci_trigger": "push", "pipeline_id": "10", "source_hash": "main@abcdef12", + "profile_summary": "fapp / detailed", + "profile_summary_meta": { + "has_profile_data": True, + "headline": "fapp / detailed", + "subline": "both, 17 runs", + "events": ["pa1", "pa2"], + "report_kinds": ["summary_text", "cpu_pa_csv"], + }, } ], pagination={"total": 1, "page": 1, "total_pages": 1}, @@ -140,9 +149,11 @@ def test_results_template_renders_table_note(): }, ) - assert "Use the server-side filters to narrow the table first" in html + assert "check the profiler summary when PA data is available" in html assert "results-table-wrap" in html assert "Compare" in html + assert "fapp / detailed" in html + assert "Profiler" in html def test_estimated_results_template_renders_table_note(): diff --git a/result_server/tests/test_result_detail_template.py b/result_server/tests/test_result_detail_template.py index cf4876b..c5633b3 100644 --- a/result_server/tests/test_result_detail_template.py +++ b/result_server/tests/test_result_detail_template.py @@ -164,6 +164,7 @@ def test_pa_data_summary_section(self, app): assert "PA Data Summary" in html assert "fapp" in html assert "single" in html + assert "single preset: pa1" in html assert "summary_text" in html assert "pa1" in html diff --git a/result_server/tests/test_results_loader.py b/result_server/tests/test_results_loader.py index be653dc..f85ef54 100644 --- a/result_server/tests/test_results_loader.py +++ b/result_server/tests/test_results_loader.py @@ -302,6 +302,7 @@ def test_existing_columns_unchanged(self, flask_app, tmp_dir): ("Nodes", "nodes"), ("P/N", "numproc_node"), ("T/P", "nthreads"), + ("Profiler", "profile_summary"), ("JSON", "json_link"), ("PA Data", "data_link"), ("Trigger", "ci_trigger"), @@ -330,6 +331,35 @@ def test_existing_row_fields_preserved(self, flask_app, tmp_dir): assert row["timestamp"] == "2025-01-01 12:00:00" assert row["numproc_node"] == "48" assert row["nthreads"] == "12" + assert row["profile_summary"] == "-" + + def test_profile_summary_is_built_from_profile_data(self, flask_app, tmp_dir): + uid = str(uuid.uuid4()) + _write_json(tmp_dir, f"result_20250101_120000_{uid}.json", { + "code": "qws", + "system": "Fugaku", + "Exp": "CASE0", + "FOM": 1.0, + "profile_data": { + "tool": "fapp", + "level": "detailed", + "report_format": "both", + "run_count": 17, + "events": [f"pa{i}" for i in range(1, 18)], + "report_kinds": ["summary_text", "cpu_pa_csv"], + }, + }) + + with flask_app.test_request_context(): + rows, _, _ = load_results_table(tmp_dir, public_only=True) + + assert len(rows) == 1 + row = rows[0] + assert row["profile_summary"] == "fapp / detailed" + assert row["profile_summary_meta"]["headline"] == "fapp / detailed" + assert row["profile_summary_meta"]["subline"] == "both, 17 runs" + assert row["profile_summary_meta"]["events"][0] == "pa1" + assert "cpu_pa_csv" in row["profile_summary_meta"]["report_kinds"] class TestSummarizeResultQuality: diff --git a/result_server/utils/results_loader.py b/result_server/utils/results_loader.py index 140a07d..48376fb 100644 --- a/result_server/utils/results_loader.py +++ b/result_server/utils/results_loader.py @@ -298,6 +298,39 @@ def _build_row(json_file, data, tgz_files): source_hash = "-" quality = summarize_result_quality(data) + profile_data = data.get("profile_data") + profile_summary = "-" + profile_summary_meta = { + "has_profile_data": False, + "headline": "", + "subline": "", + "events": [], + "report_kinds": [], + } + if isinstance(profile_data, dict) and profile_data: + tool = profile_data.get("tool") or "" + level = profile_data.get("level") or "" + report_format = profile_data.get("report_format") or "" + run_count = profile_data.get("run_count") + events = profile_data.get("events") if isinstance(profile_data.get("events"), list) else [] + report_kinds = profile_data.get("report_kinds") if isinstance(profile_data.get("report_kinds"), list) else [] + + headline_parts = [part for part in (tool, level) if part] + profile_summary = " / ".join(headline_parts) if headline_parts else "profile data" + + subline_parts = [] + if report_format: + subline_parts.append(report_format) + if isinstance(run_count, int): + subline_parts.append(f"{run_count} run" if run_count == 1 else f"{run_count} runs") + + profile_summary_meta = { + "has_profile_data": True, + "headline": profile_summary, + "subline": ", ".join(subline_parts), + "events": events, + "report_kinds": report_kinds, + } row = { "timestamp": timestamp, @@ -325,6 +358,9 @@ def _build_row(json_file, data, tgz_files): "source_info": source_info, "source_hash": source_hash, "quality": quality, + "profile_data": profile_data, + "profile_summary": profile_summary, + "profile_summary_meta": profile_summary_meta, } return row @@ -389,6 +425,7 @@ def load_results_table(directory, public_only=True, session_email=None, authenti ("Nodes", "nodes"), ("P/N", "numproc_node"), ("T/P", "nthreads"), + ("Profiler", "profile_summary"), ("JSON", "json_link"), ("PA Data", "data_link"), ("Trigger", "ci_trigger"), From aba822db5e06e72fb21022f8d5f001a045cd6213 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Mon, 13 Apr 2026 23:38:52 +0900 Subject: [PATCH 17/89] fix: refine profiler event wording --- docs/guides/add-app.md | 4 ++-- docs/guides/profiler-level-reference.md | 10 +++++----- result_server/templates/result_detail.html | 14 +++++++------- result_server/tests/test_result_detail_template.py | 3 ++- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/guides/add-app.md b/docs/guides/add-app.md index 948551e..f86f25a 100644 --- a/docs/guides/add-app.md +++ b/docs/guides/add-app.md @@ -371,8 +371,8 @@ bk_profiler_artifact/ cpu_pa_rep1.csv ``` -より一般的な profiler helper の設計方針は [Profiler Support Guide](C:/Users/yoshi/benchkit/docs/guides/profiler-support.md) を参照してください。 -level の早見表と portal 上の見え方は [Profiler Level Reference](C:/Users/yoshi/benchkit/docs/guides/profiler-level-reference.md) にまとめています。 +より一般的な profiler helper の設計方針は [Profiler Support Guide](profiler-support.md) を参照してください。 +level の早見表と portal 上の見え方は [Profiler Level Reference](profiler-level-reference.md) にまとめています。 --- diff --git a/docs/guides/profiler-level-reference.md b/docs/guides/profiler-level-reference.md index d675a81..06e2abc 100644 --- a/docs/guides/profiler-level-reference.md +++ b/docs/guides/profiler-level-reference.md @@ -18,13 +18,13 @@ These names are BenchKit-level presets. Each profiler adapter defines the concre ## Current `fapp` Mapping - `single` - - `pa1` + - event set `pa1` - `simple` - - `pa1..pa5` + - event set `pa1..pa5` - `standard` - - `pa1..pa11` + - event set `pa1..pa11` - `detailed` - - `pa1..pa17` + - event set `pa1..pa17` Default report behavior for `fapp` is: @@ -49,7 +49,7 @@ This makes it possible to inspect profiler coverage without downloading the arch - `Profiler` shows `tool / level` - the secondary line shows `report_format` and run count - Result detail - - `PA Data Summary` shows the level preset, events, and report kinds + - `PA Data Summary` shows tool-specific events, explicit events, and report kinds ## Why This Helps diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html index 8561c00..0cf3a05 100644 --- a/result_server/templates/result_detail.html +++ b/result_server/templates/result_detail.html @@ -109,23 +109,23 @@

PA Data Summary

Report Format{{ result['profile_data'].get('report_format', 'N/A') }}
Run Count{{ result['profile_data'].get('run_count', 'N/A') }}
Level SummaryTool-Specific Events {% set level = result['profile_data'].get('level') %} {% if result['profile_data'].get('tool') == 'fapp' %} {% if level == 'single' %} - single preset: pa1 + fapp event set: pa1 {% elif level == 'simple' %} - simple preset: pa1..pa5 + fapp event set: pa1..pa5 {% elif level == 'standard' %} - standard preset: pa1..pa11 + fapp event set: pa1..pa11 {% elif level == 'detailed' %} - detailed preset: pa1..pa17 + fapp event set: pa1..pa17 {% else %} - tool-specific preset + fapp tool-specific event set {% endif %} {% else %} - tool-specific preset + tool-specific event set {% endif %}
@@ -28,7 +28,7 @@ {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler", "JSON", "PA Data", "Trigger", "Pipeline", "Branch/Hash"] %} + {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "Trigger", "Pipeline", "Branch/Hash"] %} {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler", "JSON", "PA Data", "Trigger", "Pipeline", "Branch/Hash"] %} + {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "Trigger", "Pipeline", "Branch/Hash"] %} {% if col_name == "SYSTEM" %} {% for _, key in columns %} - {% if key in ["json_link", "data_link"] %} + {% if key == "json_link" %} {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "Trigger", "Pipeline", "Branch/Hash"] %} + {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "CI", "Branch/Hash"] %} {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "Trigger", "Pipeline", "Branch/Hash"] %} + {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "CI", "Branch/Hash"] %} {% if col_name == "SYSTEM" %} + {% elif key == "ci_summary" %} + {% elif key == "timestamp" %} {% set ts_parts = row[key].split(' ') %} @@ -215,7 +214,7 @@ {% else %} {% endif %} - {% elif key in ["fom_version", "exp", "ci_trigger", "pipeline_id", "source_hash"] %} + {% elif key in ["fom_version", "exp", "source_hash"] %} {% elif key == "fom" %} @@ -277,15 +276,9 @@ color: #1f4f99; } .profile-cell { - min-width: 190px; + min-width: 136px; white-space: normal; } -.profile-headline-row { - display: flex; - align-items: baseline; - justify-content: space-between; - gap: 8px; -} .profile-headline { display: block; font-weight: 600; @@ -299,7 +292,8 @@ line-height: 1.35; } .profile-data-link { - flex-shrink: 0; + display: inline-block; + margin-top: 0.18rem; font-size: 0.82em; font-weight: 600; white-space: nowrap; @@ -307,6 +301,19 @@ .profile-data-link-only { display: inline-block; } +.ci-cell { + min-width: 88px; + white-space: normal; + line-height: 1.25; +} +.ci-trigger, +.ci-pipeline { + display: block; +} +.ci-pipeline { + color: #607282; + font-size: 0.82em; +} {% include "_pagination.html" %} diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index efa68c0..0d95e98 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -94,6 +94,7 @@ def test_results_template_renders_table_note(): ("FOM", "fom"), ("Exp", "exp"), ("Profiler / PA", "profile_summary"), + ("CI", "ci_summary"), ("JSON", "json_link"), ], rows=[ @@ -154,7 +155,9 @@ def test_results_template_renders_table_note(): assert "Compare" in html assert "fapp / detailed" in html assert "Profiler / PA" in html + assert "CI" in html assert "padata" in html + assert "#10" in html def test_estimated_results_template_renders_table_note(): diff --git a/result_server/tests/test_results_loader.py b/result_server/tests/test_results_loader.py index c387b57..3cf6690 100644 --- a/result_server/tests/test_results_loader.py +++ b/result_server/tests/test_results_loader.py @@ -304,8 +304,7 @@ def test_existing_columns_unchanged(self, flask_app, tmp_dir): ("T/P", "nthreads"), ("Profiler / PA", "profile_summary"), ("JSON", "json_link"), - ("Trigger", "ci_trigger"), - ("Pipeline", "pipeline_id"), + ("CI", "ci_summary"), ] assert columns == expected_columns diff --git a/result_server/utils/results_loader.py b/result_server/utils/results_loader.py index 28ba118..60ca261 100644 --- a/result_server/utils/results_loader.py +++ b/result_server/utils/results_loader.py @@ -427,8 +427,7 @@ def load_results_table(directory, public_only=True, session_email=None, authenti ("T/P", "nthreads"), ("Profiler / PA", "profile_summary"), ("JSON", "json_link"), - ("Trigger", "ci_trigger"), - ("Pipeline", "pipeline_id"), + ("CI", "ci_summary"), ] has_filters = _has_active_filters(filter_system, filter_code, filter_exp) From 53460f51fa5a36cae4f44fe01aaeff15e7384421 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 11:14:54 +0900 Subject: [PATCH 29/89] fix: compact profiler summary cell --- result_server/templates/_results_table.html | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/result_server/templates/_results_table.html b/result_server/templates/_results_table.html index dcaaa41..4f79065 100644 --- a/result_server/templates/_results_table.html +++ b/result_server/templates/_results_table.html @@ -178,9 +178,6 @@ {% if row.profile_summary_meta.has_profile_data %} {{ row.profile_summary_meta.headline }} - {% if row.profile_summary_meta.subline %} - {{ row.profile_summary_meta.subline }} - {% endif %} {% if row.data_link %} padata {% endif %} @@ -276,7 +273,7 @@ color: #1f4f99; } .profile-cell { - min-width: 136px; + min-width: 120px; white-space: normal; } .profile-headline { @@ -284,13 +281,6 @@ font-weight: 600; line-height: 1.35; } -.profile-subline { - display: block; - margin-top: 0.15rem; - color: #607282; - font-size: 0.82em; - line-height: 1.35; -} .profile-data-link { display: inline-block; margin-top: 0.18rem; From 9164cceba22351ea123d95627ef66f57a1760992 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 11:38:00 +0900 Subject: [PATCH 30/89] refactor: split result loading responsibilities --- result_server/routes/estimated.py | 5 +- result_server/routes/results.py | 7 +- result_server/tests/test_results_loader.py | 27 +- result_server/utils/result_quality_rollup.py | 2 +- result_server/utils/result_records.py | 181 +++++ result_server/utils/result_table_rows.py | 156 ++++ result_server/utils/results_loader.py | 742 +++++-------------- 7 files changed, 561 insertions(+), 559 deletions(-) create mode 100644 result_server/utils/result_records.py create mode 100644 result_server/utils/result_table_rows.py diff --git a/result_server/routes/estimated.py b/result_server/routes/estimated.py index 9fa87e5..68fbeff 100644 --- a/result_server/routes/estimated.py +++ b/result_server/routes/estimated.py @@ -2,7 +2,8 @@ Blueprint, render_template, request, session, redirect, url_for, flash, abort, current_app, make_response ) -from utils.results_loader import load_estimated_results_table, get_filter_options, ESTIMATED_FIELD_MAP, load_single_result +from utils.result_records import load_result_json +from utils.results_loader import load_estimated_results_table, get_filter_options, ESTIMATED_FIELD_MAP from routes.results import extract_query_params from utils.user_store import get_user_store from utils.result_file import load_result_file, check_file_permission @@ -106,7 +107,7 @@ def estimated_detail(filename): abort(403, "Authentication required to view estimated data") estimated_dir = current_app.config["ESTIMATED_DIR"] check_file_permission(filename, estimated_dir) - result = load_single_result(filename, estimated_dir) + result = load_result_json(filename, estimated_dir) if result is None: abort(404, "Estimated result file not found") return render_template("estimated_detail.html", result=result, filename=filename) diff --git a/result_server/routes/results.py b/result_server/routes/results.py index 327893e..806df64 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -4,7 +4,8 @@ Blueprint, render_template, request, session, redirect, url_for, flash, abort, current_app, make_response ) -from utils.results_loader import load_results_table, load_single_result, load_multiple_results, get_filter_options, ALLOWED_PER_PAGE, DEFAULT_PER_PAGE, summarize_result_quality +from utils.result_records import load_result_json, load_result_json_batch, summarize_result_quality +from utils.results_loader import load_results_table, get_filter_options, ALLOWED_PER_PAGE, DEFAULT_PER_PAGE from utils.user_store import get_user_store from utils.app_support_matrix import load_app_system_support_matrix from utils.site_diagnostics import build_site_diagnostics @@ -179,7 +180,7 @@ def result_compare(): for filename in filenames: check_file_permission(filename, current_app.config["RECEIVED_DIR"]) - results = load_multiple_results(filenames, save_dir=current_app.config["RECEIVED_DIR"]) + results = load_result_json_batch(filenames, current_app.config["RECEIVED_DIR"]) mixed = False if results: @@ -201,7 +202,7 @@ def result_compare(): def result_detail(filename): """個別結果の詳細ページ（グラフ、データテーブル、ビルド情報）""" check_file_permission(filename, current_app.config["RECEIVED_DIR"]) - result = load_single_result(filename, save_dir=current_app.config["RECEIVED_DIR"]) + result = load_result_json(filename, current_app.config["RECEIVED_DIR"]) if result is None: abort(404, "Result file not found") quality = summarize_result_quality(result) diff --git a/result_server/tests/test_results_loader.py b/result_server/tests/test_results_loader.py index 3cf6690..7486714 100644 --- a/result_server/tests/test_results_loader.py +++ b/result_server/tests/test_results_loader.py @@ -47,7 +47,8 @@ def _setup_stubs(): sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from flask import Flask -from utils.results_loader import load_single_result, load_multiple_results, load_results_table, get_filter_options, summarize_result_quality +from utils.result_records import load_result_json, load_result_json_batch, summarize_result_quality +from utils.results_loader import load_results_table, get_filter_options # --- フィクスチャ --- @@ -92,7 +93,7 @@ def test_load_existing_file(self, tmp_dir): data = {"code": "test-app", "system": "TestSys", "FOM": 42.0} _write_json(tmp_dir, "result.json", data) - result = load_single_result("result.json", save_dir=tmp_dir) + result = load_result_json("result.json", tmp_dir) assert result is not None assert result["code"] == "test-app" assert result["system"] == "TestSys" @@ -100,7 +101,7 @@ def test_load_existing_file(self, tmp_dir): def test_load_nonexistent_file(self, tmp_dir): """存在しないファイルの場合 None を返す""" - result = load_single_result("nonexistent.json", save_dir=tmp_dir) + result = load_result_json("nonexistent.json", tmp_dir) assert result is None def test_load_invalid_json(self, tmp_dir): @@ -109,7 +110,7 @@ def test_load_invalid_json(self, tmp_dir): with open(filepath, "w") as f: f.write("not valid json {{{") - result = load_single_result("bad.json", save_dir=tmp_dir) + result = load_result_json("bad.json", tmp_dir) assert result is None def test_preserves_all_meta_fields(self, tmp_dir): @@ -125,7 +126,7 @@ def test_preserves_all_meta_fields(self, tmp_dir): } _write_json(tmp_dir, "meta.json", data) - result = load_single_result("meta.json", save_dir=tmp_dir) + result = load_result_json("meta.json", tmp_dir) assert result is not None for key in ["code", "system", "Exp", "FOM", "FOM_unit", "node_count", "cpus_per_node"]: assert key in result @@ -144,7 +145,7 @@ def test_preserves_metrics_vector(self, tmp_dir): } _write_json(tmp_dir, "vector.json", data) - result = load_single_result("vector.json", save_dir=tmp_dir) + result = load_result_json("vector.json", tmp_dir) assert result is not None assert "metrics" in result assert "vector" in result["metrics"] @@ -167,7 +168,7 @@ def test_sorts_by_timestamp_ascending(self, tmp_dir): "result_20250101_080000_bbb.json", "result_20250110_100000_ccc.json", ] - results = load_multiple_results(filenames, save_dir=tmp_dir) + results = load_result_json_batch(filenames, tmp_dir) assert len(results) == 3 assert results[0]["timestamp"] == "2025-01-01 08:00:00" @@ -180,7 +181,7 @@ def test_extracts_timestamp_from_filename(self, tmp_dir): filename = f"result_20250601_143022_{uid}.json" _write_json(tmp_dir, filename, {"code": "test"}) - results = load_multiple_results([filename], save_dir=tmp_dir) + results = load_result_json_batch([filename], tmp_dir) assert len(results) == 1 assert results[0]["timestamp"] == "2025-06-01 14:30:22" assert results[0]["filename"] == filename @@ -189,9 +190,9 @@ def test_skips_nonexistent_files(self, tmp_dir): """存在しないファイルはスキップされる""" _write_json(tmp_dir, "result_20250101_000000_x.json", {"code": "ok"}) - results = load_multiple_results( + results = load_result_json_batch( ["result_20250101_000000_x.json", "nonexistent.json"], - save_dir=tmp_dir, + directory=tmp_dir, ) assert len(results) == 1 @@ -199,7 +200,7 @@ def test_return_format(self, tmp_dir): """戻り値の形式が正しい""" _write_json(tmp_dir, "result_20250101_000000_x.json", {"code": "test", "FOM": 1.0}) - results = load_multiple_results(["result_20250101_000000_x.json"], save_dir=tmp_dir) + results = load_result_json_batch(["result_20250101_000000_x.json"], tmp_dir) assert len(results) == 1 r = results[0] assert "filename" in r @@ -212,13 +213,13 @@ def test_unknown_timestamp_for_no_pattern(self, tmp_dir): """タイムスタンプパターンがないファイル名は 'Unknown' になる""" _write_json(tmp_dir, "some_result.json", {"code": "test"}) - results = load_multiple_results(["some_result.json"], save_dir=tmp_dir) + results = load_result_json_batch(["some_result.json"], tmp_dir) assert len(results) == 1 assert results[0]["timestamp"] == "Unknown" def test_empty_filenames(self, tmp_dir): """空のファイル名リストは空リストを返す""" - results = load_multiple_results([], save_dir=tmp_dir) + results = load_result_json_batch([], tmp_dir) assert results == [] diff --git a/result_server/utils/result_quality_rollup.py b/result_server/utils/result_quality_rollup.py index 8e43c3e..7f65379 100644 --- a/result_server/utils/result_quality_rollup.py +++ b/result_server/utils/result_quality_rollup.py @@ -5,7 +5,7 @@ from datetime import datetime from utils.node_hours import extract_timestamp_from_filename -from utils.results_loader import summarize_result_quality +from utils.result_records import summarize_result_quality def _format_timestamp(filename: str, filepath: str) -> tuple[object, str]: diff --git a/result_server/utils/result_records.py b/result_server/utils/result_records.py new file mode 100644 index 0000000..07e5d1a --- /dev/null +++ b/result_server/utils/result_records.py @@ -0,0 +1,181 @@ +import json +import os +import re +from datetime import datetime + +from utils.result_file import get_file_confidential_tags + + +def load_visible_result_json( + filename, + directory, + affiliations=None, + public_only=True, + authenticated=False, +): + """Load a result JSON file after applying confidentiality checks.""" + affiliations = affiliations or [] + + tags = get_file_confidential_tags(filename, directory) + if public_only and tags: + return None + if tags and not authenticated: + return None + if tags and "admin" not in affiliations: + if not affiliations or not (set(tags) & set(affiliations)): + return None + + return load_result_json(filename, directory) + + +def load_result_json(filename, directory): + """Load a single JSON file from a result directory.""" + filepath = os.path.join(directory, filename) + if not os.path.isfile(filepath): + return None + + try: + with open(filepath, "r", encoding="utf-8") as handle: + return json.load(handle) + except Exception: + return None + + +def load_result_json_batch(filenames, directory): + """Load multiple JSON files and keep a display timestamp for compare views.""" + results = [] + for filename in filenames: + data = load_result_json(filename, directory) + if data is None: + continue + + results.append({ + "filename": filename, + "timestamp": _format_result_timestamp(filename), + "data": data, + }) + + results.sort(key=lambda item: item["timestamp"]) + return results + + +def summarize_result_quality(data): + """Summarize how estimation-ready and provenance-rich a result JSON is.""" + warnings = [] + + fom = data.get("FOM") + has_fom = fom not in (None, "", "null", "N/A") + + source_info = data.get("source_info") + has_source_info = isinstance(source_info, dict) and bool(source_info) + source_info_complete = False + if has_source_info: + source_type = source_info.get("source_type") + if source_type == "git": + source_info_complete = all(source_info.get(key) for key in ("repo_url", "branch", "commit_hash")) + elif source_type == "file": + source_info_complete = all(source_info.get(key) for key in ("file_path", "md5sum")) + else: + warnings.append("source_info has unknown source_type") + else: + warnings.append("source_info is missing") + + if has_source_info and not source_info_complete: + warnings.append("source_info is incomplete") + + fom_breakdown = data.get("fom_breakdown") + sections = [] + overlaps = [] + if isinstance(fom_breakdown, dict): + if isinstance(fom_breakdown.get("sections"), list): + sections = fom_breakdown.get("sections", []) + if isinstance(fom_breakdown.get("overlaps"), list): + overlaps = fom_breakdown.get("overlaps", []) + + has_breakdown = bool(sections or overlaps) + if not has_breakdown: + warnings.append("fom_breakdown is missing") + + section_names = [section.get("name") for section in sections if isinstance(section, dict) and section.get("name")] + if len(section_names) != len(set(section_names)): + warnings.append("duplicate section names found") + + unknown_overlap_refs = 0 + for overlap in overlaps: + if not isinstance(overlap, dict): + continue + members = overlap.get("sections", []) + if not isinstance(members, list) or not members: + warnings.append("overlap has no sections") + continue + for member in members: + if member not in section_names: + unknown_overlap_refs += 1 + + if unknown_overlap_refs: + warnings.append("overlap references undefined sections") + + section_package_count = sum( + 1 for section in sections + if isinstance(section, dict) and isinstance(section.get("estimation_package"), str) and section.get("estimation_package") + ) + overlap_package_count = sum( + 1 for overlap in overlaps + if isinstance(overlap, dict) and isinstance(overlap.get("estimation_package"), str) and overlap.get("estimation_package") + ) + artifact_count = sum( + len(item.get("artifacts", [])) + for item in sections + overlaps + if isinstance(item, dict) and isinstance(item.get("artifacts"), list) + ) + + expected_package_items = len(sections) + len(overlaps) + actual_package_items = section_package_count + overlap_package_count + estimation_ready = has_breakdown and expected_package_items > 0 and expected_package_items == actual_package_items + provenance_rich = estimation_ready and source_info_complete and artifact_count > 0 + + if has_breakdown and expected_package_items > actual_package_items: + warnings.append("some breakdown items are missing estimation_package") + + if provenance_rich: + level = "rich" + label = "Rich" + summary = "Breakdown, estimation bindings, source provenance, and artifacts are present." + elif estimation_ready: + level = "ready" + label = "Ready" + summary = "Breakdown and estimation bindings are present." + else: + level = "basic" + label = "Basic" + summary = "Core result fields are present, but estimation-related detail is limited." + + return { + "level": level, + "label": label, + "summary": summary, + "warnings": warnings, + "stats": { + "has_fom": has_fom, + "has_source_info": has_source_info, + "source_info_complete": source_info_complete, + "has_breakdown": has_breakdown, + "section_count": len(sections), + "overlap_count": len(overlaps), + "section_package_count": section_package_count, + "overlap_package_count": overlap_package_count, + "artifact_count": artifact_count, + }, + } + + +def _format_result_timestamp(filename): + match = re.search(r"\d{8}_\d{6}", filename) + if not match: + return "Unknown" + + try: + ts = datetime.strptime(match.group(), "%Y%m%d_%H%M%S") + except Exception: + return "Unknown" + return ts.strftime("%Y-%m-%d %H:%M:%S") diff --git a/result_server/utils/result_table_rows.py b/result_server/utils/result_table_rows.py new file mode 100644 index 0000000..375a7d5 --- /dev/null +++ b/result_server/utils/result_table_rows.py @@ -0,0 +1,156 @@ +import re +from datetime import datetime + +from flask import url_for + +from utils.result_records import summarize_result_quality + + +def build_result_table_row(json_filename, result_data, padata_filenames): + """Build a single row for the public/confidential results index table.""" + timestamp = _format_timestamp(json_filename) + matched_padata = _find_matching_padata_archive(json_filename, result_data, padata_filenames) + pipeline_timing = result_data.get("pipeline_timing", {}) + source_info = result_data.get("source_info") + profile_data = result_data.get("profile_data") + + ci_trigger = result_data.get("ci_trigger", "-") or "-" + pipeline_id = result_data.get("pipeline_id", "-") + if pipeline_id is None: + pipeline_id = "-" + else: + pipeline_id = str(pipeline_id) + + return { + "timestamp": timestamp, + "code": result_data.get("code", "N/A"), + "exp": result_data.get("Exp", "N/A"), + "fom": result_data.get("FOM", "N/A"), + "fom_version": result_data.get("FOM_version", "N/A"), + "system": result_data.get("system", "N/A"), + "nodes": result_data.get("node_count", "N/A"), + "numproc_node": _normalize_optional_field(result_data.get("numproc_node")), + "nthreads": _normalize_optional_field(result_data.get("nthreads")), + "json_link": url_for("results.show_result", filename=json_filename), + "data_link": url_for("results.show_result", filename=matched_padata) if matched_padata else None, + "has_vector": _has_vector_metrics(result_data), + "detail_link": url_for("results.result_detail", filename=json_filename), + "filename": json_filename, + "build_time": _normalize_pipeline_timing(pipeline_timing, "build_time"), + "queue_time": _normalize_pipeline_timing(pipeline_timing, "queue_time"), + "run_time": _normalize_pipeline_timing(pipeline_timing, "run_time"), + "execution_mode": result_data.get("execution_mode", "-") or "-", + "ci_trigger": ci_trigger, + "ci_summary": f"{ci_trigger} / {pipeline_id}", + "build_job": result_data.get("build_job", "-") or "-", + "run_job": result_data.get("run_job", "-") or "-", + "pipeline_id": pipeline_id, + "source_info": source_info, + "source_hash": _format_source_hash(source_info), + "quality": summarize_result_quality(result_data), + "profile_data": profile_data, + "profile_summary": _format_profile_summary(profile_data), + "profile_summary_meta": _build_profile_summary_meta(profile_data), + } + + +def _normalize_optional_field(value): + if value is None or value == "": + return "N/A" + return value + + +def _normalize_pipeline_timing(pipeline_timing, key): + if not isinstance(pipeline_timing, dict): + return "-" + if key not in pipeline_timing: + return "-" + return str(pipeline_timing.get(key, "-")) + + +def _has_vector_metrics(result_data): + metrics = result_data.get("metrics", {}) + return isinstance(metrics, dict) and "vector" in metrics + + +def _format_timestamp(json_filename): + match = re.search(r"\d{8}_\d{6}", json_filename) + if not match: + return "Unknown" + + try: + ts = datetime.strptime(match.group(), "%Y%m%d_%H%M%S") + except Exception: + return "Unknown" + return ts.strftime("%Y-%m-%d %H:%M:%S") + + +def _extract_result_uuid(json_filename, result_data): + uuid_match = re.search( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + json_filename, + re.IGNORECASE, + ) + if uuid_match: + return uuid_match.group(0) + return result_data.get("_server_uuid") + + +def _find_matching_padata_archive(json_filename, result_data, padata_filenames): + result_uuid = _extract_result_uuid(json_filename, result_data) + if not result_uuid: + return None + return next((filename for filename in padata_filenames if result_uuid in filename), None) + + +def _format_source_hash(source_info): + if not isinstance(source_info, dict): + return "-" + + source_type = source_info.get("source_type") + if source_type == "git": + branch = source_info.get("branch", "") + commit_hash = source_info.get("commit_hash", "") + short_hash = commit_hash[:7] if commit_hash else "" + return f"{branch}@{short_hash}" if branch and short_hash else short_hash or branch or "-" + + if source_type == "file": + md5sum = source_info.get("md5sum", "") + return md5sum[:8] if md5sum else "-" + + return "-" + + +def _format_profile_summary(profile_data): + if not isinstance(profile_data, dict) or not profile_data: + return "-" + + headline_parts = [part for part in (profile_data.get("tool"), profile_data.get("level")) if part] + return " / ".join(headline_parts) if headline_parts else "profile data" + + +def _build_profile_summary_meta(profile_data): + if not isinstance(profile_data, dict) or not profile_data: + return { + "has_profile_data": False, + "headline": "", + "subline": "", + "events": [], + "report_kinds": [], + } + + report_format = profile_data.get("report_format") or "" + run_count = profile_data.get("run_count") + subline_parts = [] + if report_format: + subline_parts.append(report_format) + if isinstance(run_count, int): + subline_parts.append(f"{run_count} run" if run_count == 1 else f"{run_count} runs") + + return { + "has_profile_data": True, + "headline": _format_profile_summary(profile_data), + "subline": ", ".join(subline_parts), + "events": profile_data.get("events") if isinstance(profile_data.get("events"), list) else [], + "report_kinds": profile_data.get("report_kinds") if isinstance(profile_data.get("report_kinds"), list) else [], + } diff --git a/result_server/utils/results_loader.py b/result_server/utils/results_loader.py index 60ca261..3281f04 100644 --- a/result_server/utils/results_loader.py +++ b/result_server/utils/results_loader.py @@ -1,418 +1,39 @@ import os -import json -import re -from datetime import datetime from math import ceil + from utils.result_file import get_file_confidential_tags -from flask import url_for +from utils.result_records import load_result_json, load_visible_result_json +from utils.result_table_rows import build_result_table_row + -# ページサイズ定数 ALLOWED_PER_PAGE = (50, 100, 200) DEFAULT_PER_PAGE = 100 -# フィールドマッピング定数 RESULT_FIELD_MAP = {"system": "system", "code": "code", "exp": "Exp"} ESTIMATED_FIELD_MAP = {"system": "current_system.system", "code": "code", "exp": "exp"} -#-------------------------------------------------------------------------------------------------------------- -def load_json_with_confidential_filter(json_file, directory, affs=None, public_only=True, authenticated=False): - """ - 指定ディレクトリの JSON を読み込み、confidential タグに基づいて - フィルタリングする。 - - Args: - json_file (str): JSON ファイル名 - directory (str): ファイルのあるディレクトリ - affs (list, optional): セッションユーザーの所属リスト - public_only (bool): 公開のみかどうか - authenticated (bool): 認証済みかどうか - - Returns: - dict or None: 読み込んだ JSON データ（フィルタに引っかかれば None） - """ - if affs is None: - affs = [] - - tags = get_file_confidential_tags(json_file, directory) - if public_only and tags: - return None - if tags and not authenticated: - return None - if tags and "admin" not in affs: - if not affs or not (set(tags) & set(affs)): - return None - - try: - with open(os.path.join(directory, json_file), "r", encoding="utf-8") as f: - data = json.load(f) - return data - except Exception: - return None - -#-------------------------------------------------------------------------------------------------------------- - -def load_single_result(filename, save_dir): - """ - 指定ファイル名のJSONを読み込み dict で返す。 - ファイルが存在しない場合は None を返す。 - 権限チェックはルート側で実施するため、ここでは単純なJSON読み込みのみ行う。 - - Args: - filename (str): JSONファイル名 - save_dir (str): ファイルのあるディレクトリ（必須） - - Returns: - dict or None: 読み込んだJSONデータ、ファイルが存在しない場合はNone - """ - filepath = os.path.join(save_dir, filename) - if not os.path.isfile(filepath): - return None - try: - with open(filepath, "r", encoding="utf-8") as f: - return json.load(f) - except Exception: - return None - - -def summarize_result_quality(data): - """Result JSON の品質サマリを返す。""" - warnings = [] - - fom = data.get("FOM") - has_fom = fom not in (None, "", "null", "N/A") - - source_info = data.get("source_info") - has_source_info = isinstance(source_info, dict) and bool(source_info) - source_info_complete = False - if has_source_info: - source_type = source_info.get("source_type") - if source_type == "git": - source_info_complete = all(source_info.get(k) for k in ("repo_url", "branch", "commit_hash")) - elif source_type == "file": - source_info_complete = all(source_info.get(k) for k in ("file_path", "md5sum")) - else: - warnings.append("source_info has unknown source_type") - else: - warnings.append("source_info is missing") - - if has_source_info and not source_info_complete: - warnings.append("source_info is incomplete") - - fom_breakdown = data.get("fom_breakdown") - sections = [] - overlaps = [] - if isinstance(fom_breakdown, dict): - if isinstance(fom_breakdown.get("sections"), list): - sections = fom_breakdown.get("sections", []) - if isinstance(fom_breakdown.get("overlaps"), list): - overlaps = fom_breakdown.get("overlaps", []) - - has_breakdown = bool(sections or overlaps) - if not has_breakdown: - warnings.append("fom_breakdown is missing") - - section_names = [s.get("name") for s in sections if isinstance(s, dict) and s.get("name")] - if len(section_names) != len(set(section_names)): - warnings.append("duplicate section names found") - - unknown_overlap_refs = 0 - for overlap in overlaps: - if not isinstance(overlap, dict): - continue - members = overlap.get("sections", []) - if not isinstance(members, list) or not members: - warnings.append("overlap has no sections") - continue - for member in members: - if member not in section_names: - unknown_overlap_refs += 1 - - if unknown_overlap_refs: - warnings.append("overlap references undefined sections") - - section_pkg_count = sum( - 1 for s in sections - if isinstance(s, dict) and isinstance(s.get("estimation_package"), str) and s.get("estimation_package") - ) - overlap_pkg_count = sum( - 1 for o in overlaps - if isinstance(o, dict) and isinstance(o.get("estimation_package"), str) and o.get("estimation_package") - ) - artifact_count = sum( - len(item.get("artifacts", [])) - for item in sections + overlaps - if isinstance(item, dict) and isinstance(item.get("artifacts"), list) - ) - - expected_pkg_items = len(sections) + len(overlaps) - actual_pkg_items = section_pkg_count + overlap_pkg_count - estimation_ready = has_breakdown and expected_pkg_items > 0 and expected_pkg_items == actual_pkg_items - provenance_rich = estimation_ready and source_info_complete and artifact_count > 0 - - if has_breakdown and expected_pkg_items > actual_pkg_items: - warnings.append("some breakdown items are missing estimation_package") - - if provenance_rich: - level = "rich" - label = "Rich" - summary = "Breakdown, estimation bindings, source provenance, and artifacts are present." - elif estimation_ready: - level = "ready" - label = "Ready" - summary = "Breakdown and estimation bindings are present." - else: - level = "basic" - label = "Basic" - summary = "Core result fields are present, but estimation-related detail is limited." - - return { - "level": level, - "label": label, - "summary": summary, - "warnings": warnings, - "stats": { - "has_fom": has_fom, - "has_source_info": has_source_info, - "source_info_complete": source_info_complete, - "has_breakdown": has_breakdown, - "section_count": len(sections), - "overlap_count": len(overlaps), - "section_package_count": section_pkg_count, - "overlap_package_count": overlap_pkg_count, - "artifact_count": artifact_count, - }, - } - - -def load_multiple_results(filenames, save_dir): - """ - 複数の結果JSONを読み込み、タイムスタンプ昇順でソートしたリストを返す。 - - Args: - filenames (list[str]): JSONファイル名のリスト - save_dir (str): ファイルのあるディレクトリ（必須） - - Returns: - list[dict]: [{"filename": str, "timestamp": str, "data": dict}, ...] - タイムスタンプ昇順でソート済み - """ - results = [] - for filename in filenames: - data = load_single_result(filename, save_dir) - if data is None: - continue - - # ファイル名から YYYYMMDD_HHMMSS パターンでタイムスタンプを抽出 - timestamp = "Unknown" - match = re.search(r"\d{8}_\d{6}", filename) - if match: - try: - ts = datetime.strptime(match.group(), "%Y%m%d_%H%M%S") - timestamp = ts.strftime("%Y-%m-%d %H:%M:%S") - except Exception: - pass - - results.append({ - "filename": filename, - "timestamp": timestamp, - "data": data, - }) - - # タイムスタンプ昇順でソート（"Unknown" は先頭に来る） - results.sort(key=lambda r: r["timestamp"]) - return results - -def _build_row(json_file, data, tgz_files): - """JSONデータからテーブル行を構築するヘルパー関数""" - code = data.get("code", "N/A") - sys_name = data.get("system", "N/A") - fom = data.get("FOM", "N/A") - fom_version = data.get("FOM_version", "N/A") - exp = data.get("Exp", "N/A") - nodes = data.get("node_count", "N/A") - - numproc_node = data.get("numproc_node", "N/A") - if numproc_node is None or numproc_node == "": - numproc_node = "N/A" - - nthreads = data.get("nthreads", "N/A") - if nthreads is None or nthreads == "": - nthreads = "N/A" - - # get timestamp and uuid - match = re.search(r"\d{8}_\d{6}", json_file) - timestamp = "Unknown" - if match: - try: - ts = datetime.strptime(match.group(), "%Y%m%d_%H%M%S") - timestamp = ts.strftime("%Y-%m-%d %H:%M:%S") - except Exception: - pass - - uuid_match = re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", json_file, re.IGNORECASE) - uid = uuid_match.group(0) if uuid_match else data.get("_server_uuid") - tgz_file = next((f for f in tgz_files if uid in f), None) if uid else None - - # metrics.vector の有無を判定 - metrics = data.get("metrics", {}) - has_vector = isinstance(metrics, dict) and "vector" in metrics - - # pipeline_timing fields - pipeline_timing = data.get("pipeline_timing", None) - if isinstance(pipeline_timing, dict): - build_time = str(pipeline_timing.get("build_time", "-")) if "build_time" in pipeline_timing else "-" - queue_time = str(pipeline_timing.get("queue_time", "-")) if "queue_time" in pipeline_timing else "-" - run_time = str(pipeline_timing.get("run_time", "-")) if "run_time" in pipeline_timing else "-" - else: - build_time = "-" - queue_time = "-" - run_time = "-" - - execution_mode = data.get("execution_mode", "-") or "-" - ci_trigger = data.get("ci_trigger", "-") or "-" - build_job = data.get("build_job", "-") or "-" - run_job = data.get("run_job", "-") or "-" - pipeline_id = data.get("pipeline_id", "-") - if pipeline_id is None: - pipeline_id = "-" - else: - pipeline_id = str(pipeline_id) - - # source_info の読み込み - source_info = data.get("source_info", None) - - # source_hash の生成（Branch/Hash列用） - if source_info and isinstance(source_info, dict): - st = source_info.get("source_type") - if st == "git": - _branch = source_info.get("branch", "") - commit = source_info.get("commit_hash", "") - short_hash = commit[:7] if commit else "" - source_hash = f"{_branch}@{short_hash}" if _branch and short_hash else short_hash or _branch or "-" - elif st == "file": - md5 = source_info.get("md5sum", "") - source_hash = md5[:8] if md5 else "-" - else: - source_hash = "-" - else: - source_hash = "-" - - quality = summarize_result_quality(data) - profile_data = data.get("profile_data") - profile_summary = "-" - profile_summary_meta = { - "has_profile_data": False, - "headline": "", - "subline": "", - "events": [], - "report_kinds": [], - } - if isinstance(profile_data, dict) and profile_data: - tool = profile_data.get("tool") or "" - level = profile_data.get("level") or "" - report_format = profile_data.get("report_format") or "" - run_count = profile_data.get("run_count") - events = profile_data.get("events") if isinstance(profile_data.get("events"), list) else [] - report_kinds = profile_data.get("report_kinds") if isinstance(profile_data.get("report_kinds"), list) else [] - - headline_parts = [part for part in (tool, level) if part] - profile_summary = " / ".join(headline_parts) if headline_parts else "profile data" - - subline_parts = [] - if report_format: - subline_parts.append(report_format) - if isinstance(run_count, int): - subline_parts.append(f"{run_count} run" if run_count == 1 else f"{run_count} runs") - - profile_summary_meta = { - "has_profile_data": True, - "headline": profile_summary, - "subline": ", ".join(subline_parts), - "events": events, - "report_kinds": report_kinds, - } - - row = { - "timestamp": timestamp, - "code": code, - "exp": exp, - "fom": fom, - "fom_version": fom_version, - "system": sys_name, - "nodes": nodes, - "numproc_node": numproc_node, - "nthreads": nthreads, - "json_link": url_for("results.show_result", filename=json_file), - "data_link": url_for("results.show_result", filename=tgz_file) if tgz_file else None, - "has_vector": has_vector, - "detail_link": url_for("results.result_detail", filename=json_file), - "filename": json_file, - "build_time": build_time, - "queue_time": queue_time, - "run_time": run_time, - "execution_mode": execution_mode, - "ci_trigger": ci_trigger, - "build_job": build_job, - "run_job": run_job, - "pipeline_id": pipeline_id, - "source_info": source_info, - "source_hash": source_hash, - "quality": quality, - "profile_data": profile_data, - "profile_summary": profile_summary, - "profile_summary_meta": profile_summary_meta, - } - return row - - -def _has_active_filters(filter_system, filter_code, filter_exp): - """フィルタが1つでも指定されているか判定""" - return any(f is not None for f in (filter_system, filter_code, filter_exp)) - - -def _get_nested(data, field_path): - """ドット記法のフィールドパスでネストされた値を取得する。""" - keys = field_path.split(".") - val = data - for k in keys: - if isinstance(val, dict): - val = val.get(k) - else: - return None - return val - - -def _matches_filters(data, filter_system, filter_code, filter_exp, field_map=None): - """フィールドマッピングに基づいてフィルタ条件を判定する。""" - if field_map is None: - field_map = RESULT_FIELD_MAP - if filter_system is not None: - val = _get_nested(data, field_map["system"]) - if val != filter_system: - return False - if filter_code is not None: - val = _get_nested(data, field_map["code"]) - if val != filter_code: - return False - if filter_exp is not None: - val = _get_nested(data, field_map["exp"]) - if val != filter_exp: - return False - return True - -def load_results_table(directory, public_only=True, session_email=None, authenticated=False, affiliations=None, - page=1, per_page=100, filter_system=None, filter_code=None, filter_exp=None, - padata_directory=None): - # per_page バリデーション +def load_results_table( + directory, + public_only=True, + session_email=None, + authenticated=False, + affiliations=None, + page=1, + per_page=100, + filter_system=None, + filter_code=None, + filter_exp=None, + padata_directory=None, +): if per_page not in ALLOWED_PER_PAGE: per_page = DEFAULT_PER_PAGE - affs = affiliations if affiliations is not None else [] + affiliations = affiliations if affiliations is not None else [] files = os.listdir(directory) - json_files = sorted([f for f in files if f.endswith(".json")], reverse=True) - tgz_dir = padata_directory or directory - tgz_files = [f for f in os.listdir(tgz_dir) if f.endswith(".tgz")] + json_filenames = sorted([filename for filename in files if filename.endswith(".json")], reverse=True) + padata_dir = padata_directory or directory + padata_filenames = [filename for filename in os.listdir(padata_dir) if filename.endswith(".tgz")] columns = [ ("Timestamp", "timestamp"), @@ -430,100 +51,103 @@ def load_results_table(directory, public_only=True, session_email=None, authenti ("CI", "ci_summary"), ] - has_filters = _has_active_filters(filter_system, filter_code, filter_exp) - - if not has_filters: - # フィルタなし: ファイル名リストに対して confidential フィルタを適用後、スライスして対象ページのJSONのみ読み込み - # confidential フィルタはJSONの中身ではなくファイル名ベースなので、先に適用可能なファイルを特定 - # ただし confidential チェックにはファイル読み込みが不要（タグはファイル名から判定） - # load_json_with_confidential_filter が None を返すファイルを除外する必要があるため、 - # まず有効なファイルリストを構築してからスライスする - valid_json_files = [] - for json_file in json_files: - tags = get_file_confidential_tags(json_file, directory) - if public_only and tags: - continue - if tags and not authenticated: - continue - if tags and "admin" not in affs: - if not affs or not (set(tags) & set(affs)): - continue - valid_json_files.append(json_file) - - paginated_files, pagination_info = paginate_list(valid_json_files, page, per_page) - + filters_are_active = _filters_are_active(filter_system, filter_code, filter_exp) + + if not filters_are_active: + visible_filenames = [ + filename + for filename in json_filenames + if _is_result_filename_visible( + filename, + directory, + affiliations, + public_only, + authenticated, + ) + ] + + paginated_filenames, pagination_info = paginate_list(visible_filenames, page, per_page) rows = [] - for json_file in paginated_files: - try: - with open(os.path.join(directory, json_file), "r", encoding="utf-8") as f: - data = json.load(f) - except Exception: + for filename in paginated_filenames: + result_data = load_result_json(filename, directory) + if result_data is None: continue - rows.append(_build_row(json_file, data, tgz_files)) + rows.append(build_result_table_row(filename, result_data, padata_filenames)) return rows, columns, pagination_info - else: - # フィルタあり: 全JSON読み込み後にフィルタ適用、結果リストをスライス - rows = [] - for json_file in json_files: - data = load_json_with_confidential_filter(json_file, directory, affs, public_only, authenticated) - if data is None: - continue - if not _matches_filters(data, filter_system, filter_code, filter_exp, field_map=RESULT_FIELD_MAP): - continue - rows.append(_build_row(json_file, data, tgz_files)) - paginated_rows, pagination_info = paginate_list(rows, page, per_page) - return paginated_rows, columns, pagination_info + rows = [] + for filename in json_filenames: + result_data = load_visible_result_json( + filename, + directory, + affiliations, + public_only, + authenticated, + ) + if result_data is None: + continue + if not _matches_table_filters(result_data, filter_system, filter_code, filter_exp, field_map=RESULT_FIELD_MAP): + continue + rows.append(build_result_table_row(filename, result_data, padata_filenames)) + + paginated_rows, pagination_info = paginate_list(rows, page, per_page) + return paginated_rows, columns, pagination_info -def load_estimated_results_table(directory, public_only=True, session_email=None, authenticated=False, affiliations=None, - page=1, per_page=100, filter_system=None, filter_code=None, filter_exp=None): - # per_page バリデーション +def load_estimated_results_table( + directory, + public_only=True, + session_email=None, + authenticated=False, + affiliations=None, + page=1, + per_page=100, + filter_system=None, + filter_code=None, + filter_exp=None, +): if per_page not in ALLOWED_PER_PAGE: per_page = DEFAULT_PER_PAGE - affs = affiliations if affiliations is not None else [] + affiliations = affiliations if affiliations is not None else [] files = os.listdir(directory) - json_files = sorted([f for f in files if f.endswith(".json")], reverse=True) - - has_filters = _has_active_filters(filter_system, filter_code, filter_exp) + json_filenames = sorted([filename for filename in files if filename.endswith(".json")], reverse=True) + filters_are_active = _filters_are_active(filter_system, filter_code, filter_exp) rows = [] - for json_file in json_files: - data = load_json_with_confidential_filter(json_file, directory, affs, public_only, authenticated) - if data is None: + for filename in json_filenames: + result_data = load_visible_result_json( + filename, + directory, + affiliations, + public_only, + authenticated, + ) + if result_data is None: continue - if has_filters and not _matches_filters(data, filter_system, filter_code, filter_exp, field_map=ESTIMATED_FIELD_MAP): + if filters_are_active and not _matches_table_filters( + result_data, + filter_system, + filter_code, + filter_exp, + field_map=ESTIMATED_FIELD_MAP, + ): continue - current = data.get("current_system", {}) - future = data.get("future_system", {}) - estimate_meta = data.get("estimate_metadata", {}) - applicability = data.get("applicability", {}) - - # get timestamp and uuid - match = re.search(r"\d{8}_\d{6}", json_file) - timestamp = "Unknown" - if match: - try: - ts = datetime.strptime(match.group(), "%Y%m%d_%H%M%S") - timestamp = ts.strftime("%Y-%m-%d %H:%M:%S") - except Exception: - pass - - uuid_match = re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", json_file, re.IGNORECASE) - uid = uuid_match.group(0) if uuid_match else None - - estimate_timestamp = estimate_meta.get("estimation_result_timestamp") or timestamp - estimate_uuid = estimate_meta.get("estimation_result_uuid") or uid - - row = { - "timestamp": estimate_timestamp, - "code": data.get("code", ""), - "exp": data.get("exp", ""), - # System A (current_system) + current = result_data.get("current_system", {}) + future = result_data.get("future_system", {}) + estimate_meta = result_data.get("estimate_metadata", {}) + applicability = result_data.get("applicability", {}) + + timestamp = _format_timestamp_from_filename(filename) + estimate_uuid = estimate_meta.get("estimation_result_uuid") or _extract_uuid_from_filename(filename) + + rows.append({ + "timestamp": estimate_meta.get("estimation_result_timestamp") or timestamp, + "code": result_data.get("code", ""), + "exp": result_data.get("exp", ""), "systemA_system": current.get("system", ""), "systemA_fom": current.get("fom", ""), "systemA_target_nodes": current.get("target_nodes", ""), @@ -531,7 +155,6 @@ def load_estimated_results_table(directory, public_only=True, session_email=None "systemA_bench_system": current.get("benchmark", {}).get("system", ""), "systemA_bench_fom": current.get("benchmark", {}).get("fom", ""), "systemA_bench_nodes": current.get("benchmark", {}).get("nodes", ""), - # System B (future_system) "systemB_system": future.get("system", ""), "systemB_fom": future.get("fom", ""), "systemB_target_nodes": future.get("target_nodes", ""), @@ -539,7 +162,6 @@ def load_estimated_results_table(directory, public_only=True, session_email=None "systemB_bench_system": future.get("benchmark", {}).get("system", ""), "systemB_bench_fom": future.get("benchmark", {}).get("fom", ""), "systemB_bench_nodes": future.get("benchmark", {}).get("nodes", ""), - # Common "applicability_status": applicability.get("status", ""), "requested_estimation_package": estimate_meta.get("requested_estimation_package", ""), "estimation_package": estimate_meta.get("estimation_package", ""), @@ -550,16 +172,14 @@ def load_estimated_results_table(directory, public_only=True, session_email=None "requested_current_estimation_package": estimate_meta.get("current_package", {}).get("requested_estimation_package", ""), "requested_future_estimation_package": estimate_meta.get("future_package", {}).get("requested_estimation_package", ""), "estimate_uuid": estimate_uuid or "", - "performance_ratio": data.get("performance_ratio", ""), - "json_link": json_file, - } - rows.append(row) + "performance_ratio": result_data.get("performance_ratio", ""), + "json_link": filename, + }) columns = [ ("Timestamp", "timestamp"), ("CODE", "code"), ("Exp", "exp"), - # System A (current_system) ("A System", "systemA_system"), ("A FOM", "systemA_fom"), ("A Target Nodes", "systemA_target_nodes"), @@ -567,7 +187,6 @@ def load_estimated_results_table(directory, public_only=True, session_email=None ("A Bench System", "systemA_bench_system"), ("A Bench FOM", "systemA_bench_fom"), ("A Bench Nodes", "systemA_bench_nodes"), - # System B (future_system) ("B System", "systemB_system"), ("B FOM", "systemB_fom"), ("B Target Nodes", "systemB_target_nodes"), @@ -575,7 +194,6 @@ def load_estimated_results_table(directory, public_only=True, session_email=None ("B Bench System", "systemB_bench_system"), ("B Bench FOM", "systemB_bench_fom"), ("B Bench Nodes", "systemB_bench_nodes"), - # Common ("Applicability", "applicability_status"), ("Requested Package", "requested_estimation_package"), ("Applied Package", "estimation_package"), @@ -588,83 +206,62 @@ def load_estimated_results_table(directory, public_only=True, session_email=None return paginated_rows, columns, pagination_info -def get_filter_options(directory, public_only=True, authenticated=False, affiliations=None, - field_map=None, filter_code=None): - """ - 全JSONファイルからフィルタドロップダウンの選択肢を抽出して返す。 - field_map に基づいて systems/codes/exps のフィールド名を動的に参照する。 - - Args: - directory (str): JSONファイルのあるディレクトリ - public_only (bool): 公開のみかどうか - authenticated (bool): 認証済みかどうか - affiliations (list, optional): セッションユーザーの所属リスト - field_map (dict, optional): フィールド名マッピング。None の場合は RESULT_FIELD_MAP を使用。 - - Returns: - dict: {"systems": sorted_list, "codes": sorted_list, "exps": sorted_list} - """ +def get_filter_options( + directory, + public_only=True, + authenticated=False, + affiliations=None, + field_map=None, + filter_code=None, +): if field_map is None: field_map = RESULT_FIELD_MAP - affs = affiliations if affiliations is not None else [] + + affiliations = affiliations if affiliations is not None else [] systems = set() codes = set() - exps = set() + experiments = set() try: files = os.listdir(directory) except OSError: return {"systems": [], "codes": [], "exps": []} - json_files = [f for f in files if f.endswith(".json")] - - for json_file in json_files: - data = load_json_with_confidential_filter(json_file, directory, affs, public_only, authenticated) - if data is None: + for filename in [name for name in files if name.endswith(".json")]: + result_data = load_visible_result_json( + filename, + directory, + affiliations, + public_only, + authenticated, + ) + if result_data is None: continue - val = _get_nested(data, field_map["system"]) - if val and val != "N/A": - systems.add(val) + system = _get_nested_field(result_data, field_map["system"]) + if system and system != "N/A": + systems.add(system) - val = _get_nested(data, field_map["code"]) - if val and val != "N/A": - codes.add(val) + code = _get_nested_field(result_data, field_map["code"]) + if code and code != "N/A": + codes.add(code) - val = _get_nested(data, field_map["exp"]) - if val and val != "N/A": - if not filter_code or _get_nested(data, field_map["code"]) == filter_code: - exps.add(val) + exp = _get_nested_field(result_data, field_map["exp"]) + if exp and exp != "N/A": + if not filter_code or _get_nested_field(result_data, field_map["code"]) == filter_code: + experiments.add(exp) return { "systems": sorted(systems), "codes": sorted(codes), - "exps": sorted(exps), + "exps": sorted(experiments), } -def paginate_list(items: list, page: int, per_page: int) -> tuple[list, dict]: - """ - リストにページネーションを適用する。 - - Args: - items: ページネーション対象のリスト - page: ページ番号（1始まり） - per_page: 1ページあたりの件数 - - Returns: - (paginated_items, pagination_info) - pagination_info = { - "page": int, - "per_page": int, - "total": int, - "total_pages": int, - } - """ +def paginate_list(items, page, per_page): total = len(items) total_pages = max(1, ceil(total / per_page)) if total > 0 else 1 - # クランプ: 範囲外のページ番号を有効範囲に収める if page < 1: page = 1 elif page > total_pages: @@ -680,5 +277,70 @@ def paginate_list(items: list, page: int, per_page: int) -> tuple[list, dict]: "total": total, "total_pages": total_pages, } - return paginated_items, pagination_info + + +def _is_result_filename_visible(filename, directory, affiliations, public_only, authenticated): + tags = get_file_confidential_tags(filename, directory) + if public_only and tags: + return False + if tags and not authenticated: + return False + if tags and "admin" not in affiliations: + if not affiliations or not (set(tags) & set(affiliations)): + return False + return True + + +def _filters_are_active(filter_system, filter_code, filter_exp): + return any(value is not None for value in (filter_system, filter_code, filter_exp)) + + +def _get_nested_field(data, field_path): + keys = field_path.split(".") + value = data + for key in keys: + if isinstance(value, dict): + value = value.get(key) + else: + return None + return value + + +def _matches_table_filters(data, filter_system, filter_code, filter_exp, field_map=None): + if field_map is None: + field_map = RESULT_FIELD_MAP + + if filter_system is not None and _get_nested_field(data, field_map["system"]) != filter_system: + return False + if filter_code is not None and _get_nested_field(data, field_map["code"]) != filter_code: + return False + if filter_exp is not None and _get_nested_field(data, field_map["exp"]) != filter_exp: + return False + return True + + +def _format_timestamp_from_filename(filename): + import re + from datetime import datetime + + match = re.search(r"\d{8}_\d{6}", filename) + if not match: + return "Unknown" + + try: + ts = datetime.strptime(match.group(), "%Y%m%d_%H%M%S") + except Exception: + return "Unknown" + return ts.strftime("%Y-%m-%d %H:%M:%S") + + +def _extract_uuid_from_filename(filename): + import re + + uuid_match = re.search( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + filename, + re.IGNORECASE, + ) + return uuid_match.group(0) if uuid_match else None From 266f1a42898683fb47a5767905b7acb9214f5cac Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 11:50:05 +0900 Subject: [PATCH 31/89] refactor: split result table partials --- result_server/templates/_results_table.html | 54 ++----------------- .../templates/_results_table_cell_ci.html | 4 ++ .../templates/_results_table_cell_code.html | 7 +++ .../_results_table_cell_profile.html | 21 ++++++++ .../templates/_results_table_cell_system.html | 18 +++++++ 5 files changed, 54 insertions(+), 50 deletions(-) create mode 100644 result_server/templates/_results_table_cell_ci.html create mode 100644 result_server/templates/_results_table_cell_code.html create mode 100644 result_server/templates/_results_table_cell_profile.html create mode 100644 result_server/templates/_results_table_cell_system.html diff --git a/result_server/templates/_results_table.html b/result_server/templates/_results_table.html index 4f79065..0fe4b62 100644 --- a/result_server/templates/_results_table.html +++ b/result_server/templates/_results_table.html @@ -153,64 +153,18 @@ {% endif %} {% elif key == "system" %} - + {% include "_results_table_cell_system.html" %} {% elif key in ["nodes", "numproc_node", "nthreads"] %} {{ row[key] }} {% elif key == "profile_summary" %} - + {% include "_results_table_cell_profile.html" %} {% elif key == "ci_summary" %} - + {% include "_results_table_cell_ci.html" %} {% elif key == "timestamp" %} {% set ts_parts = row[key].split(' ') %} {% elif key == "code" %} - {% if row.source_info and row.source_info.source_type == "git" %} - - {% elif row.source_info and row.source_info.source_type == "file" %} - - {% else %} - - {% endif %} + {% include "_results_table_cell_code.html" %} {% elif key in ["fom_version", "exp", "source_hash"] %} {% elif key == "fom" %} diff --git a/result_server/templates/_results_table_cell_ci.html b/result_server/templates/_results_table_cell_ci.html new file mode 100644 index 0000000..df51656 --- /dev/null +++ b/result_server/templates/_results_table_cell_ci.html @@ -0,0 +1,4 @@ + diff --git a/result_server/templates/_results_table_cell_code.html b/result_server/templates/_results_table_cell_code.html new file mode 100644 index 0000000..ff18fed --- /dev/null +++ b/result_server/templates/_results_table_cell_code.html @@ -0,0 +1,7 @@ +{% if row.source_info and row.source_info.source_type == "git" %} + +{% elif row.source_info and row.source_info.source_type == "file" %} + +{% else %} + +{% endif %} diff --git a/result_server/templates/_results_table_cell_profile.html b/result_server/templates/_results_table_cell_profile.html new file mode 100644 index 0000000..619a031 --- /dev/null +++ b/result_server/templates/_results_table_cell_profile.html @@ -0,0 +1,21 @@ + diff --git a/result_server/templates/_results_table_cell_system.html b/result_server/templates/_results_table_cell_system.html new file mode 100644 index 0000000..e25e495 --- /dev/null +++ b/result_server/templates/_results_table_cell_system.html @@ -0,0 +1,18 @@ + From 8c9b99859ca890f27a25ae5b8eea5b04a11c70de Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 11:57:47 +0900 Subject: [PATCH 32/89] refactor: make result table headers data-driven --- result_server/templates/_results_table.html | 79 +++++-------------- .../tests/test_portal_list_templates.py | 16 ++-- result_server/tests/test_results_loader.py | 26 +++--- result_server/utils/results_loader.py | 32 ++++---- 4 files changed, 56 insertions(+), 97 deletions(-) diff --git a/result_server/templates/_results_table.html b/result_server/templates/_results_table.html index 0fe4b62..37dc4a2 100644 --- a/result_server/templates/_results_table.html +++ b/result_server/templates/_results_table.html @@ -27,59 +27,17 @@ - {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "CI", "Branch/Hash"] %} - - {% endif %} - {% if col_name == "FOM" %} + {% for column in columns %} + + {% if column.label == "FOM" %} - {% for col_name, _ in columns %} - {% if col_name in ["Timestamp", "SYSTEM", "CODE", "FOM", "FOM version", "Exp", "Nodes", "P/N", "T/P", "Profiler / PA", "JSON", "CI", "Branch/Hash"] %} - {% if col_name == "SYSTEM" %} + {% for column in columns %} + {% if column.label == "SYSTEM" %} - {% elif col_name == "CODE" %} + {% elif column.label == "CODE" %} - {% elif col_name == "Exp" %} + {% elif column.label == "Exp" %} {% endif %} - {% endif %} - {% if col_name == "FOM" %} + {% if column.label == "FOM" %} {% for row in rows %} - {% for _, key in columns %} + {% for column in columns %} + {% set key = column.key %} {% if key == "json_link" %} +{% elif key == "json_link" %} + +{% else %} + +{% endif %} diff --git a/result_server/templates/estimated_results.html b/result_server/templates/estimated_results.html index e0250a0..f9020d3 100644 --- a/result_server/templates/estimated_results.html +++ b/result_server/templates/estimated_results.html @@ -55,69 +55,7 @@

Focus on system pairs, applied packages, and ratio at the list level, then open detail when you need breakdowns or fallback/applicability context.

-
-{% set leading_columns = columns | selectattr("section", "equalto", "leading") | list %} -{% set trailing_columns = columns | selectattr("section", "equalto", "trailing") | list %} -{% set system_a_columns = columns | selectattr("group", "equalto", "System A") | list %} -{% set system_b_columns = columns | selectattr("group", "equalto", "System B") | list %} -
{% if col_name == "Timestamp" %} {{ col_name }} @@ -62,13 +62,9 @@ {{ col_name }} Version identifier for the FOM measurement section - helps identify which code region was measured when users modify the timing boundaries - {% elif col_name == "PA Data" %} - {{ col_name }} - Performance Analysis data archive (detailed profiling data) - - {% elif col_name == "Profiler" %} + {% elif col_name == "Profiler / PA" %} {{ col_name }} - Profiler tool, level, and report summary extracted from uploaded PA metadata + Profiler tool, level, report summary, and PA data download access {% elif col_name == "Trigger" %} {{ col_name }} @@ -106,7 +102,7 @@
- {% if row[key] %} - - {{ "json" if key == "json_link" else "data" }} - + {% if row.json_link %} + json {% else %} - {% endif %} @@ -187,17 +181,25 @@ {% if row.profile_summary_meta.has_profile_data %} - {{ row.profile_summary_meta.headline }} + + {{ row.profile_summary_meta.headline }} + {% if row.data_link %} + padata + {% endif %} + {% if row.profile_summary_meta.subline %} {{ row.profile_summary_meta.subline }} {% endif %} {{ row.profile_summary_meta.headline }} {% if row.profile_summary_meta.subline %}
{{ row.profile_summary_meta.subline }}{% endif %} + {% if row.data_link %}
archive: available{% endif %} {% if row.profile_summary_meta.events %}
events: {{ row.profile_summary_meta.events | join(', ') }}{% endif %} {% if row.profile_summary_meta.report_kinds %}
reports: {{ row.profile_summary_meta.report_kinds | join(', ') }}{% endif %} + {% elif row.data_link %} + padata {% else %} - {% endif %} @@ -275,9 +277,15 @@ color: #1f4f99; } .profile-cell { - min-width: 150px; + min-width: 190px; white-space: normal; } +.profile-headline-row { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px; +} .profile-headline { display: block; font-weight: 600; @@ -290,6 +298,15 @@ font-size: 0.82em; line-height: 1.35; } +.profile-data-link { + flex-shrink: 0; + font-size: 0.82em; + font-weight: 600; + white-space: nowrap; +} +.profile-data-link-only { + display: inline-block; +} {% include "_pagination.html" %} diff --git a/result_server/templates/estimated_results.html b/result_server/templates/estimated_results.html index 319047f..49cc7c5 100644 --- a/result_server/templates/estimated_results.html +++ b/result_server/templates/estimated_results.html @@ -1,7 +1,7 @@ {% extends "_results_base.html" %} {% block title %}Estimated Results{% endblock %} -{% block page_subtitle %}Review projected performance across systems and estimation packages. Estimated data requires authentication.{% endblock %} +{% block page_subtitle %}Review projected performance and open detail for breakdowns or applicability. Estimated data requires authentication.{% endblock %} {% block auth_warning %} {% if not authenticated %} diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index 799f1d5..efa68c0 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -93,7 +93,7 @@ def test_results_template_renders_table_note(): ("CODE", "code"), ("FOM", "fom"), ("Exp", "exp"), - ("Profiler", "profile_summary"), + ("Profiler / PA", "profile_summary"), ("JSON", "json_link"), ], rows=[ @@ -104,7 +104,7 @@ def test_results_template_renders_table_note(): "fom": 1.234, "exp": "CASE0", "json_link": "/results/result0.json", - "data_link": None, + "data_link": "/results/padata0.tgz", "filename": "result0.json", "detail_link": "/results/detail/result0.json", "source_info": None, @@ -149,11 +149,12 @@ def test_results_template_renders_table_note(): }, ) - assert "check the profiler summary when PA data is available" in html + assert "check the profiler and PA summary when available" in html assert "results-table-wrap" in html assert "Compare" in html assert "fapp / detailed" in html - assert "Profiler" in html + assert "Profiler / PA" in html + assert "padata" in html def test_estimated_results_template_renders_table_note(): diff --git a/result_server/tests/test_results_loader.py b/result_server/tests/test_results_loader.py index f85ef54..c387b57 100644 --- a/result_server/tests/test_results_loader.py +++ b/result_server/tests/test_results_loader.py @@ -302,9 +302,8 @@ def test_existing_columns_unchanged(self, flask_app, tmp_dir): ("Nodes", "nodes"), ("P/N", "numproc_node"), ("T/P", "nthreads"), - ("Profiler", "profile_summary"), + ("Profiler / PA", "profile_summary"), ("JSON", "json_link"), - ("PA Data", "data_link"), ("Trigger", "ci_trigger"), ("Pipeline", "pipeline_id"), ] diff --git a/result_server/utils/results_loader.py b/result_server/utils/results_loader.py index 48376fb..28ba118 100644 --- a/result_server/utils/results_loader.py +++ b/result_server/utils/results_loader.py @@ -425,9 +425,8 @@ def load_results_table(directory, public_only=True, session_email=None, authenti ("Nodes", "nodes"), ("P/N", "numproc_node"), ("T/P", "nthreads"), - ("Profiler", "profile_summary"), + ("Profiler / PA", "profile_summary"), ("JSON", "json_link"), - ("PA Data", "data_link"), ("Trigger", "ci_trigger"), ("Pipeline", "pipeline_id"), ] From d03cfe41a7040f660b34b6391ed3213c9f9b2713 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 11:10:28 +0900 Subject: [PATCH 28/89] fix: compact profiler and CI columns --- result_server/templates/_results_table.html | 53 +++++++++++-------- .../tests/test_portal_list_templates.py | 3 ++ result_server/tests/test_results_loader.py | 3 +- result_server/utils/results_loader.py | 3 +- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/result_server/templates/_results_table.html b/result_server/templates/_results_table.html index 984e603..dcaaa41 100644 --- a/result_server/templates/_results_table.html +++ b/result_server/templates/_results_table.html @@ -28,7 +28,7 @@
{% if col_name == "Timestamp" %} {{ col_name }} @@ -66,13 +66,9 @@ {{ col_name }} Profiler tool, level, report summary, and PA data download access - {% elif col_name == "Trigger" %} + {% elif col_name == "CI" %} {{ col_name }} - CI pipeline trigger source (schedule, push, web, etc.) - - {% elif col_name == "Pipeline" %} - {{ col_name }} - GitLab CI pipeline ID + CI trigger source and pipeline ID {% elif col_name == "Branch/Hash" %} {{ col_name }} @@ -102,7 +98,7 @@
{% if row.profile_summary_meta.has_profile_data %} - - {{ row.profile_summary_meta.headline }} - {% if row.data_link %} - padata - {% endif %} - + {{ row.profile_summary_meta.headline }} {% if row.profile_summary_meta.subline %} {{ row.profile_summary_meta.subline }} {% endif %} + {% if row.data_link %} + padata + {% endif %} {{ row.profile_summary_meta.headline }} {% if row.profile_summary_meta.subline %}
{{ row.profile_summary_meta.subline }}{% endif %} @@ -204,6 +198,11 @@ - {% endif %} 		+ {{ row.ci_trigger }} + #{{ row.pipeline_id }} + {{ ts_parts[0] }}
{{ ts_parts[1] if ts_parts|length > 1 else '' }}		{{ row.code }}{{ row.quality.label }}{{ row[key] }}{{ "%.3f"|format(row[key]|float) if row[key] not in [None, '', 'N/A', 'null', 'nan'] else row[key] }} - {{ row[key] }} - - {% set sys_info = systems_info.get(row[key]) %} - {% if sys_info %} - {{ sys_info.name }}
- CPU: {{ sys_info.cpu_name }}
- CPU/node: {{ sys_info.cpu_per_node }}
- CPU Cores: {{ sys_info.cpu_cores }}
- {% if sys_info.gpu_name != '-' %}GPU: {{ sys_info.gpu_name }}
- GPU/node: {{ sys_info.gpu_per_node }}
{% endif %} - Memory: {{ sys_info.memory }} - {% else %} - {{ row[key] }} system details - {% endif %} - - - 		- {% if row.profile_summary_meta.has_profile_data %} - - {{ row.profile_summary_meta.headline }} - {% if row.data_link %} - padata - {% endif %} - - {{ row.profile_summary_meta.headline }} - {% if row.profile_summary_meta.subline %}
{{ row.profile_summary_meta.subline }}{% endif %} - {% if row.data_link %}
archive: available{% endif %} - {% if row.profile_summary_meta.events %}
events: {{ row.profile_summary_meta.events | join(', ') }}{% endif %} - {% if row.profile_summary_meta.report_kinds %}
reports: {{ row.profile_summary_meta.report_kinds | join(', ') }}{% endif %} - - - {% elif row.data_link %} - padata - {% else %} - - - {% endif %} - 		- {{ row.ci_trigger }} - #{{ row.pipeline_id }} - {{ ts_parts[0] }}
{{ ts_parts[1] if ts_parts|length > 1 else '' }}		{{ row.code }}{{ row.quality.label }}{{ row.code }}{{ row.quality.label }}{{ row.code }}{{ row.quality.label }}{{ row[key] }} + {{ row.ci_trigger }} + #{{ row.pipeline_id }} +{{ row.code }}{{ row.quality.label }}{{ row.code }}{{ row.quality.label }}{{ row.code }}{{ row.quality.label }} + {% if row.profile_summary_meta.has_profile_data %} + + {{ row.profile_summary_meta.headline }} + {% if row.data_link %} + padata + {% endif %} + + {{ row.profile_summary_meta.headline }} + {% if row.profile_summary_meta.subline %}
{{ row.profile_summary_meta.subline }}{% endif %} + {% if row.data_link %}
archive: available{% endif %} + {% if row.profile_summary_meta.events %}
events: {{ row.profile_summary_meta.events | join(', ') }}{% endif %} + {% if row.profile_summary_meta.report_kinds %}
reports: {{ row.profile_summary_meta.report_kinds | join(', ') }}{% endif %} + + + {% elif row.data_link %} + padata + {% else %} + - + {% endif %} +		 + {{ row.system }} + + {% set sys_info = systems_info.get(row.system) %} + {% if sys_info %} + {{ sys_info.name }}
+ CPU: {{ sys_info.cpu_name }}
+ CPU/node: {{ sys_info.cpu_per_node }}
+ CPU Cores: {{ sys_info.cpu_cores }}
+ {% if sys_info.gpu_name != '-' %}GPU: {{ sys_info.gpu_name }}
+ GPU/node: {{ sys_info.gpu_per_node }}
{% endif %} + Memory: {{ sys_info.memory }} + {% else %} + {{ row.system }} system details + {% endif %} + + +
- {% if col_name == "Timestamp" %} - {{ col_name }} - Date and time when benchmark execution completed and results were automatically submitted to server - - {% elif col_name == "SYSTEM" %} - {{ col_name }} - Computing system name - - {% elif col_name == "FOM" %} - {{ col_name }} - Figure of Merit - Benchmark performance metric value, typically elapsed time in seconds for main section - - {% elif col_name == "Exp" %} - {{ col_name }} - Experimental conditions (filtered by CODE) - - {% elif col_name == "P/N" %} - {{ col_name }} - Number of processes per node - - {% elif col_name == "T/P" %} - {{ col_name }} - Number of threads per process - - {% elif col_name == "JSON" %} - {{ col_name }} - Detailed benchmark results in JSON format - - {% elif col_name == "FOM version" %} - {{ col_name }} - Version identifier for the FOM measurement section - helps identify which code region was measured when users modify the timing boundaries - - {% elif col_name == "Profiler / PA" %} - {{ col_name }} - Profiler tool, level, report summary, and PA data download access - - {% elif col_name == "CI" %} - {{ col_name }} - CI trigger source and pipeline ID - - {% elif col_name == "Branch/Hash" %} - {{ col_name }} - Source code branch name and short commit hash (git) or short md5sum (file archive) - - {% else %} - {{ col_name }} - {% endif %} - + {% if column.tooltip %} + {{ column.label }} + {{ column.tooltip }} + + {% else %} + {{ column.label }} + {% endif %} + @@ -97,9 +55,8 @@
@@ -143,7 +99,8 @@
{% if row.json_link %} diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index 0d95e98..8f0a7dc 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -88,14 +88,14 @@ def test_results_template_renders_table_note(): html = render_template( "results.html", columns=[ - ("Timestamp", "timestamp"), - ("SYSTEM", "system"), - ("CODE", "code"), - ("FOM", "fom"), - ("Exp", "exp"), - ("Profiler / PA", "profile_summary"), - ("CI", "ci_summary"), - ("JSON", "json_link"), + {"label": "Timestamp", "key": "timestamp", "tooltip": "Date and time when benchmark execution completed and results were automatically submitted to server", "tooltip_class": "tooltip-left"}, + {"label": "SYSTEM", "key": "system", "tooltip": "Computing system name"}, + {"label": "CODE", "key": "code"}, + {"label": "FOM", "key": "fom", "tooltip": "Figure of Merit - Benchmark performance metric value, typically elapsed time in seconds for main section"}, + {"label": "Exp", "key": "exp", "tooltip": "Experimental conditions (filtered by CODE)"}, + {"label": "Profiler / PA", "key": "profile_summary", "tooltip": "Profiler tool, level, report summary, and PA data download access"}, + {"label": "CI", "key": "ci_summary", "tooltip": "CI trigger source and pipeline ID"}, + {"label": "JSON", "key": "json_link", "tooltip": "Detailed benchmark results in JSON format", "tooltip_class": "tooltip-right"}, ], rows=[ { diff --git a/result_server/tests/test_results_loader.py b/result_server/tests/test_results_loader.py index 7486714..00938f4 100644 --- a/result_server/tests/test_results_loader.py +++ b/result_server/tests/test_results_loader.py @@ -293,19 +293,19 @@ def test_existing_columns_unchanged(self, flask_app, tmp_dir): rows, columns, pagination_info = load_results_table(tmp_dir, public_only=True) expected_columns = [ - ("Timestamp", "timestamp"), - ("CODE", "code"), - ("Branch/Hash", "source_hash"), - ("Exp", "exp"), - ("FOM", "fom"), - ("FOM version", "fom_version"), - ("SYSTEM", "system"), - ("Nodes", "nodes"), - ("P/N", "numproc_node"), - ("T/P", "nthreads"), - ("Profiler / PA", "profile_summary"), - ("JSON", "json_link"), - ("CI", "ci_summary"), + {"label": "Timestamp", "key": "timestamp", "tooltip": "Date and time when benchmark execution completed and results were automatically submitted to server", "tooltip_class": "tooltip-left"}, + {"label": "CODE", "key": "code"}, + {"label": "Branch/Hash", "key": "source_hash", "tooltip": "Source code branch name and short commit hash (git) or short md5sum (file archive)"}, + {"label": "Exp", "key": "exp", "tooltip": "Experimental conditions (filtered by CODE)"}, + {"label": "FOM", "key": "fom", "tooltip": "Figure of Merit - Benchmark performance metric value, typically elapsed time in seconds for main section"}, + {"label": "FOM version", "key": "fom_version", "tooltip": "Version identifier for the FOM measurement section - helps identify which code region was measured when users modify the timing boundaries"}, + {"label": "SYSTEM", "key": "system", "tooltip": "Computing system name"}, + {"label": "Nodes", "key": "nodes"}, + {"label": "P/N", "key": "numproc_node", "tooltip": "Number of processes per node"}, + {"label": "T/P", "key": "nthreads", "tooltip": "Number of threads per process"}, + {"label": "Profiler / PA", "key": "profile_summary", "tooltip": "Profiler tool, level, report summary, and PA data download access"}, + {"label": "JSON", "key": "json_link", "tooltip": "Detailed benchmark results in JSON format", "tooltip_class": "tooltip-right"}, + {"label": "CI", "key": "ci_summary", "tooltip": "CI trigger source and pipeline ID"}, ] assert columns == expected_columns diff --git a/result_server/utils/results_loader.py b/result_server/utils/results_loader.py index 3281f04..4edadb9 100644 --- a/result_server/utils/results_loader.py +++ b/result_server/utils/results_loader.py @@ -12,6 +12,22 @@ RESULT_FIELD_MAP = {"system": "system", "code": "code", "exp": "Exp"} ESTIMATED_FIELD_MAP = {"system": "current_system.system", "code": "code", "exp": "exp"} +RESULT_TABLE_COLUMNS = [ + {"label": "Timestamp", "key": "timestamp", "tooltip": "Date and time when benchmark execution completed and results were automatically submitted to server", "tooltip_class": "tooltip-left"}, + {"label": "CODE", "key": "code"}, + {"label": "Branch/Hash", "key": "source_hash", "tooltip": "Source code branch name and short commit hash (git) or short md5sum (file archive)"}, + {"label": "Exp", "key": "exp", "tooltip": "Experimental conditions (filtered by CODE)"}, + {"label": "FOM", "key": "fom", "tooltip": "Figure of Merit - Benchmark performance metric value, typically elapsed time in seconds for main section"}, + {"label": "FOM version", "key": "fom_version", "tooltip": "Version identifier for the FOM measurement section - helps identify which code region was measured when users modify the timing boundaries"}, + {"label": "SYSTEM", "key": "system", "tooltip": "Computing system name"}, + {"label": "Nodes", "key": "nodes"}, + {"label": "P/N", "key": "numproc_node", "tooltip": "Number of processes per node"}, + {"label": "T/P", "key": "nthreads", "tooltip": "Number of threads per process"}, + {"label": "Profiler / PA", "key": "profile_summary", "tooltip": "Profiler tool, level, report summary, and PA data download access"}, + {"label": "JSON", "key": "json_link", "tooltip": "Detailed benchmark results in JSON format", "tooltip_class": "tooltip-right"}, + {"label": "CI", "key": "ci_summary", "tooltip": "CI trigger source and pipeline ID"}, +] + def load_results_table( directory, @@ -35,21 +51,7 @@ def load_results_table( padata_dir = padata_directory or directory padata_filenames = [filename for filename in os.listdir(padata_dir) if filename.endswith(".tgz")] - columns = [ - ("Timestamp", "timestamp"), - ("CODE", "code"), - ("Branch/Hash", "source_hash"), - ("Exp", "exp"), - ("FOM", "fom"), - ("FOM version", "fom_version"), - ("SYSTEM", "system"), - ("Nodes", "nodes"), - ("P/N", "numproc_node"), - ("T/P", "nthreads"), - ("Profiler / PA", "profile_summary"), - ("JSON", "json_link"), - ("CI", "ci_summary"), - ] + columns = RESULT_TABLE_COLUMNS filters_are_active = _filters_are_active(filter_system, filter_code, filter_exp) From 6287866cb53d4481423ca51df9370a15e8a9d1d6 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 12:09:56 +0900 Subject: [PATCH 33/89] refactor: simplify estimated result table rendering --- .../templates/estimated_results.html | 160 ++++++----------- .../tests/test_estimated_results_loader.py | 8 + .../tests/test_portal_list_templates.py | 42 +++++ result_server/utils/estimated_table_rows.py | 169 ++++++++++++++++++ result_server/utils/result_records.py | 13 +- result_server/utils/results_loader.py | 111 ++---------- 6 files changed, 301 insertions(+), 202 deletions(-) create mode 100644 result_server/utils/estimated_table_rows.py diff --git a/result_server/templates/estimated_results.html b/result_server/templates/estimated_results.html index 49cc7c5..e0250a0 100644 --- a/result_server/templates/estimated_results.html +++ b/result_server/templates/estimated_results.html @@ -42,133 +42,77 @@ .estimated-link-cell { white-space: nowrap; } + .estimated-code-cell { + font-family: monospace; + font-size: 0.9em; + } + .estimated-meta-note { + font-size: 10px; + line-height: 1.25; + color: #6b7280; + }

Focus on system pairs, applied packages, and ratio at the list level, then open detail when you need breakdowns or fallback/applicability context.

+{% set leading_columns = columns | selectattr("section", "equalto", "leading") | list %} +{% set trailing_columns = columns | selectattr("section", "equalto", "trailing") | list %} +{% set system_a_columns = columns | selectattr("group", "equalto", "System A") | list %} +{% set system_b_columns = columns | selectattr("group", "equalto", "System B") | list %} - - - - - - - - - - - + {% for column in leading_columns %} + + {% endfor %} + {% if system_a_columns %} + + {% endif %} + {% if system_b_columns %} + + {% endif %} + {% for column in trailing_columns %} + + {% endfor %} - - - - - - - - - - - - - - + {% for column in system_a_columns %} + + {% endfor %} + {% for column in system_b_columns %} + + {% endfor %} {% for r in rows %} - {% set ts_parts = r.timestamp.split(' ') %} - {% set a_fom = "%.3f"|format(r.systemA_fom|float) if r.systemA_fom not in [None, '', 'N/A', 'null', 'nan'] else r.systemA_fom %} - {% set b_fom = "%.3f"|format(r.systemB_fom|float) if r.systemB_fom not in [None, '', 'N/A', 'null', 'nan'] else r.systemB_fom %} - {% set ratio = "%.3f"|format(r.performance_ratio|float) if r.performance_ratio not in [None, '', 'N/A', 'null', 'nan'] else r.performance_ratio %} - {% set a_scaling_full = r.systemA_scaling_method %} - {% set b_scaling_full = r.systemB_scaling_method %} - {% set requested_full = r.requested_estimation_package %} - {% set applied_full = r.estimation_package %} - {% if a_scaling_full == 'instrumented-app-sections-dummy' %} - {% set a_scaling_short = 'instr-app-sec' %} - {% elif a_scaling_full == 'scale-mock' %} - {% set a_scaling_short = 'scale-mock' %} - {% else %} - {% set a_scaling_short = a_scaling_full %} - {% endif %} - {% if b_scaling_full == 'instrumented-app-sections-dummy' %} - {% set b_scaling_short = 'instr-app-sec' %} - {% elif b_scaling_full == 'scale-mock' %} - {% set b_scaling_short = 'scale-mock' %} - {% else %} - {% set b_scaling_short = b_scaling_full %} - {% endif %} - {% if requested_full == 'instrumented_app_sections_dummy' %} - {% set requested_short = 'instr_app_sec' %} - {% elif requested_full == 'lightweight_fom_scaling' %} - {% set requested_short = 'weakscaling' %} - {% elif requested_full == 'weakscaling' %} - {% set requested_short = 'weakscaling' %} - {% else %} - {% set requested_short = requested_full %} - {% endif %} - {% if applied_full == 'instrumented_app_sections_dummy' %} - {% set applied_short = 'instr_app_sec' %} - {% elif applied_full == 'lightweight_fom_scaling' %} - {% set applied_short = 'weakscaling' %} - {% elif applied_full == 'weakscaling' %} - {% set applied_short = 'weakscaling' %} - {% else %} - {% set applied_short = applied_full %} - {% endif %} - {% set requested_title = requested_full %} - {% if r.requested_current_estimation_package %} - {% set requested_title = requested_title ~ '\ncurrent-side: ' ~ r.requested_current_estimation_package %} - {% endif %} - {% if r.requested_future_estimation_package %} - {% set requested_title = requested_title ~ '\nfuture-side: ' ~ r.requested_future_estimation_package %} - {% endif %} - {% set applied_title = applied_full %} - {% if r.method_class %} - {% set applied_title = applied_title ~ '\nclass: ' ~ r.method_class %} - {% endif %} - {% if r.detail_level %} - {% set applied_title = applied_title ~ '\ndetail: ' ~ r.detail_level %} - {% endif %} - {% if r.current_estimation_package %} - {% set applied_title = applied_title ~ '\ncurrent-side: ' ~ r.current_estimation_package %} - {% endif %} - {% if r.future_estimation_package %} - {% set applied_title = applied_title ~ '\nfuture-side: ' ~ r.future_estimation_package %} - {% endif %} - - - - - - - - - - - - - - - - - - - - - - - + {% elif key == "json_link" %} + + {% else %} + + {% endif %} + {% endfor %} {% endfor %} diff --git a/result_server/tests/test_estimated_results_loader.py b/result_server/tests/test_estimated_results_loader.py index cac53b2..be73c73 100644 --- a/result_server/tests/test_estimated_results_loader.py +++ b/result_server/tests/test_estimated_results_loader.py @@ -90,9 +90,17 @@ def test_estimated_rows_prefer_metadata_fields(flask_app, tmp_dir): assert info["total"] == 1 assert rows[0]["timestamp"] == "2026-04-06 12:34:56" + assert rows[0]["timestamp_date"] == "2026-04-06" + assert rows[0]["timestamp_time"] == "12:34:56" assert rows[0]["estimate_uuid"] == "11111111-2222-3333-4444-555555555555" + assert rows[0]["estimate_uuid_short"] == "11111111" assert rows[0]["requested_estimation_package"] == "instrumented_app_sections_dummy" assert rows[0]["estimation_package"] == "weakscaling" + assert rows[0]["requested_package_short"] == "instr_app_sec" + assert rows[0]["applied_package_short"] == "weakscaling" + assert rows[0]["systemA_fom_display"] == "1.000" + assert rows[0]["systemB_fom_display"] == "2.000" + assert rows[0]["performance_ratio_display"] == "2.000" assert rows[0]["method_class"] == "minimum" assert rows[0]["detail_level"] == "basic" assert rows[0]["current_estimation_package"] == "weakscaling" diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index 8f0a7dc..2884bf1 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -168,36 +168,78 @@ def test_estimated_results_template_renders_table_note(): html = render_template( "estimated_results.html", authenticated=True, + columns=[ + {"label": "Timestamp", "key": "timestamp", "section": "leading"}, + {"label": "CODE", "key": "code", "section": "leading"}, + {"label": "Exp", "key": "exp", "section": "leading"}, + {"label": "System", "key": "systemA_system", "group": "System A"}, + {"label": "FOM", "key": "systemA_fom_display", "group": "System A", "title_key": "systemA_fom", "align": "right"}, + {"label": "Target Nodes", "key": "systemA_target_nodes", "group": "System A"}, + {"label": "Scaling Method", "key": "systemA_scaling_short", "group": "System A", "title_key": "systemA_scaling_title"}, + {"label": "Bench System", "key": "systemA_bench_system", "group": "System A"}, + {"label": "Bench FOM", "key": "systemA_bench_fom_display", "group": "System A", "title_key": "systemA_bench_fom", "align": "right"}, + {"label": "Bench Nodes", "key": "systemA_bench_nodes", "group": "System A"}, + {"label": "System", "key": "systemB_system", "group": "System B"}, + {"label": "FOM", "key": "systemB_fom_display", "group": "System B", "title_key": "systemB_fom", "align": "right"}, + {"label": "Target Nodes", "key": "systemB_target_nodes", "group": "System B"}, + {"label": "Scaling Method", "key": "systemB_scaling_short", "group": "System B", "title_key": "systemB_scaling_title"}, + {"label": "Bench System", "key": "systemB_bench_system", "group": "System B"}, + {"label": "Bench FOM", "key": "systemB_bench_fom_display", "group": "System B", "title_key": "systemB_bench_fom", "align": "right"}, + {"label": "Bench Nodes", "key": "systemB_bench_nodes", "group": "System B"}, + {"label": "Applicability", "key": "applicability_status", "section": "trailing"}, + {"label": "Requested Package", "key": "requested_package_short", "section": "trailing", "title_key": "requested_package_title"}, + {"label": "Applied Package", "key": "applied_package_short", "section": "trailing", "title_key": "applied_package_title", "meta_key": "applied_package_meta_line"}, + {"label": "Estimate UUID", "key": "estimate_uuid_short", "section": "trailing", "title_key": "estimate_uuid", "cell_class": "estimated-code-cell"}, + {"label": "Ratio", "key": "performance_ratio_display", "section": "trailing", "title_key": "performance_ratio", "align": "right"}, + {"label": "JSON", "key": "json_link", "section": "trailing", "cell_class": "estimated-link-cell"}, + ], rows=[ { "timestamp": "2026-04-13 12:00:00", + "timestamp_date": "2026-04-13", + "timestamp_time": "12:00:00", "code": "qws", "exp": "CASE0", "systemA_system": "Fugaku", "systemA_fom": 0.944, + "systemA_fom_display": "0.944", "systemA_target_nodes": "1024", "systemA_scaling_method": "weakscaling", + "systemA_scaling_short": "weakscaling", + "systemA_scaling_title": "weakscaling", "systemA_bench_system": "Fugaku", "systemA_bench_fom": 0.386, + "systemA_bench_fom_display": "0.386", "systemA_bench_nodes": "1", "systemB_system": "FugakuNEXT", "systemB_fom": 9.054, + "systemB_fom_display": "9.054", "systemB_target_nodes": "256", "systemB_scaling_method": "instrumented-app-sections-dummy", + "systemB_scaling_short": "instr-app-sec", + "systemB_scaling_title": "instrumented-app-sections-dummy", "systemB_bench_system": "MiyabiG", "systemB_bench_fom": 5.712, + "systemB_bench_fom_display": "5.712", "systemB_bench_nodes": "1", "applicability_status": "applicable", "requested_estimation_package": "instrumented_app_sections_dummy", "estimation_package": "instrumented_app_sections_dummy", + "requested_package_short": "instr_app_sec", + "applied_package_short": "instr_app_sec", "requested_current_estimation_package": "weakscaling", "requested_future_estimation_package": "instrumented_app_sections_dummy", "current_estimation_package": "weakscaling", "future_estimation_package": "instrumented_app_sections_dummy", "method_class": "detailed", "detail_level": "intermediate", + "requested_package_title": "instrumented_app_sections_dummy\ncurrent-side: weakscaling\nfuture-side: instrumented_app_sections_dummy", + "applied_package_title": "instrumented_app_sections_dummy\nclass: detailed\ndetail: intermediate\ncurrent-side: weakscaling\nfuture-side: instrumented_app_sections_dummy", + "applied_package_meta_line": "detailed / intermediate", "estimate_uuid": "11111111-2222-3333-4444-555555555555", + "estimate_uuid_short": "11111111", "performance_ratio": 0.104, + "performance_ratio_display": "0.104", "json_link": "estimate0.json", } ], diff --git a/result_server/utils/estimated_table_rows.py b/result_server/utils/estimated_table_rows.py new file mode 100644 index 0000000..a0af06c --- /dev/null +++ b/result_server/utils/estimated_table_rows.py @@ -0,0 +1,169 @@ +def build_estimated_table_row(filename, result_data, fallback_uuid=None, fallback_timestamp=None): + current = result_data.get("current_system", {}) + future = result_data.get("future_system", {}) + estimate_meta = result_data.get("estimate_metadata", {}) + applicability = result_data.get("applicability", {}) + + estimate_uuid = estimate_meta.get("estimation_result_uuid") or fallback_uuid or "" + timestamp = estimate_meta.get("estimation_result_timestamp") or fallback_timestamp or "" + + requested_package = estimate_meta.get("requested_estimation_package", "") + applied_package = estimate_meta.get("estimation_package", "") + method_class = estimate_meta.get("method_class", "") + detail_level = estimate_meta.get("detail_level", "") + requested_current = estimate_meta.get("current_package", {}).get("requested_estimation_package", "") + requested_future = estimate_meta.get("future_package", {}).get("requested_estimation_package", "") + current_package = estimate_meta.get("current_package", {}).get("estimation_package", "") + future_package = estimate_meta.get("future_package", {}).get("estimation_package", "") + + return { + "timestamp": timestamp, + "timestamp_date": _get_timestamp_date(timestamp), + "timestamp_time": _get_timestamp_time(timestamp), + "code": result_data.get("code", ""), + "exp": result_data.get("exp", ""), + "systemA_system": current.get("system", ""), + "systemA_fom": current.get("fom", ""), + "systemA_fom_display": _format_numeric_display(current.get("fom", "")), + "systemA_target_nodes": current.get("target_nodes", ""), + "systemA_scaling_method": current.get("scaling_method", ""), + "systemA_scaling_title": current.get("scaling_method", ""), + "systemA_bench_system": current.get("benchmark", {}).get("system", ""), + "systemA_bench_fom": current.get("benchmark", {}).get("fom", ""), + "systemA_bench_fom_display": _format_numeric_display(current.get("benchmark", {}).get("fom", "")), + "systemA_bench_nodes": current.get("benchmark", {}).get("nodes", ""), + "systemB_system": future.get("system", ""), + "systemB_fom": future.get("fom", ""), + "systemB_fom_display": _format_numeric_display(future.get("fom", "")), + "systemB_target_nodes": future.get("target_nodes", ""), + "systemB_scaling_method": future.get("scaling_method", ""), + "systemB_scaling_title": future.get("scaling_method", ""), + "systemB_bench_system": future.get("benchmark", {}).get("system", ""), + "systemB_bench_fom": future.get("benchmark", {}).get("fom", ""), + "systemB_bench_fom_display": _format_numeric_display(future.get("benchmark", {}).get("fom", "")), + "systemB_bench_nodes": future.get("benchmark", {}).get("nodes", ""), + "applicability_status": applicability.get("status", ""), + "requested_estimation_package": requested_package, + "estimation_package": applied_package, + "method_class": method_class, + "detail_level": detail_level, + "current_estimation_package": current_package, + "future_estimation_package": future_package, + "requested_current_estimation_package": requested_current, + "requested_future_estimation_package": requested_future, + "estimate_uuid": estimate_uuid, + "estimate_uuid_short": estimate_uuid[:8] if estimate_uuid else "", + "performance_ratio": result_data.get("performance_ratio", ""), + "performance_ratio_display": _format_numeric_display(result_data.get("performance_ratio", "")), + "json_link": filename, + "requested_package_short": _format_package_short_name(requested_package), + "applied_package_short": _format_package_short_name(applied_package), + "systemA_scaling_short": _format_scaling_short_name(current.get("scaling_method", "")), + "systemB_scaling_short": _format_scaling_short_name(future.get("scaling_method", "")), + "requested_package_title": _build_requested_package_title( + requested_package, + requested_current, + requested_future, + ), + "applied_package_title": _build_applied_package_title( + applied_package, + method_class, + detail_level, + current_package, + future_package, + ), + "applied_package_meta_line": _build_applied_package_meta_line(method_class, detail_level), + } + + +def build_estimated_table_columns(): + return [ + {"label": "Timestamp", "key": "timestamp", "section": "leading"}, + {"label": "CODE", "key": "code", "section": "leading"}, + {"label": "Exp", "key": "exp", "section": "leading"}, + {"label": "System", "key": "systemA_system", "group": "System A"}, + {"label": "FOM", "key": "systemA_fom_display", "group": "System A", "title_key": "systemA_fom", "align": "right"}, + {"label": "Target Nodes", "key": "systemA_target_nodes", "group": "System A"}, + {"label": "Scaling Method", "key": "systemA_scaling_short", "group": "System A", "title_key": "systemA_scaling_title"}, + {"label": "Bench System", "key": "systemA_bench_system", "group": "System A"}, + {"label": "Bench FOM", "key": "systemA_bench_fom_display", "group": "System A", "title_key": "systemA_bench_fom", "align": "right"}, + {"label": "Bench Nodes", "key": "systemA_bench_nodes", "group": "System A"}, + {"label": "System", "key": "systemB_system", "group": "System B"}, + {"label": "FOM", "key": "systemB_fom_display", "group": "System B", "title_key": "systemB_fom", "align": "right"}, + {"label": "Target Nodes", "key": "systemB_target_nodes", "group": "System B"}, + {"label": "Scaling Method", "key": "systemB_scaling_short", "group": "System B", "title_key": "systemB_scaling_title"}, + {"label": "Bench System", "key": "systemB_bench_system", "group": "System B"}, + {"label": "Bench FOM", "key": "systemB_bench_fom_display", "group": "System B", "title_key": "systemB_bench_fom", "align": "right"}, + {"label": "Bench Nodes", "key": "systemB_bench_nodes", "group": "System B"}, + {"label": "Applicability", "key": "applicability_status", "section": "trailing"}, + {"label": "Requested Package", "key": "requested_package_short", "section": "trailing", "title_key": "requested_package_title"}, + {"label": "Applied Package", "key": "applied_package_short", "section": "trailing", "title_key": "applied_package_title", "meta_key": "applied_package_meta_line"}, + {"label": "Estimate UUID", "key": "estimate_uuid_short", "section": "trailing", "title_key": "estimate_uuid", "cell_class": "estimated-code-cell"}, + {"label": "Ratio", "key": "performance_ratio_display", "section": "trailing", "title_key": "performance_ratio", "align": "right"}, + {"label": "JSON", "key": "json_link", "section": "trailing", "cell_class": "estimated-link-cell"}, + ] + + +def _format_scaling_short_name(value): + if value == "instrumented-app-sections-dummy": + return "instr-app-sec" + if value == "scale-mock": + return "scale-mock" + return value + + +def _format_package_short_name(value): + if value == "instrumented_app_sections_dummy": + return "instr_app_sec" + if value in ("lightweight_fom_scaling", "weakscaling"): + return "weakscaling" + return value + + +def _build_requested_package_title(requested_package, requested_current, requested_future): + title = requested_package or "" + if requested_current: + title += "\ncurrent-side: " + requested_current + if requested_future: + title += "\nfuture-side: " + requested_future + return title + + +def _build_applied_package_title(applied_package, method_class, detail_level, current_package, future_package): + title = applied_package or "" + if method_class: + title += "\nclass: " + method_class + if detail_level: + title += "\ndetail: " + detail_level + if current_package: + title += "\ncurrent-side: " + current_package + if future_package: + title += "\nfuture-side: " + future_package + return title + + +def _build_applied_package_meta_line(method_class, detail_level): + meta_parts = [part for part in (method_class, detail_level) if part] + return " / ".join(meta_parts) + + +def _format_numeric_display(value): + if value in (None, "", "N/A", "null", "nan"): + return value + + try: + return f"{float(value):.3f}" + except (TypeError, ValueError): + return value + + +def _get_timestamp_date(value): + if not value: + return "" + return value.split(" ")[0] + + +def _get_timestamp_time(value): + if not value or " " not in value: + return "" + return value.split(" ", 1)[1] diff --git a/result_server/utils/result_records.py b/result_server/utils/result_records.py index 07e5d1a..0e8e818 100644 --- a/result_server/utils/result_records.py +++ b/result_server/utils/result_records.py @@ -51,7 +51,7 @@ def load_result_json_batch(filenames, directory): results.append({ "filename": filename, - "timestamp": _format_result_timestamp(filename), + "timestamp": format_result_timestamp(filename), "data": data, }) @@ -169,7 +169,7 @@ def summarize_result_quality(data): } -def _format_result_timestamp(filename): +def format_result_timestamp(filename): match = re.search(r"\d{8}_\d{6}", filename) if not match: return "Unknown" @@ -179,3 +179,12 @@ def _format_result_timestamp(filename): except Exception: return "Unknown" return ts.strftime("%Y-%m-%d %H:%M:%S") + + +def extract_result_uuid(filename): + uuid_match = re.search( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + filename, + re.IGNORECASE, + ) + return uuid_match.group(0) if uuid_match else None diff --git a/result_server/utils/results_loader.py b/result_server/utils/results_loader.py index 4edadb9..3b18f72 100644 --- a/result_server/utils/results_loader.py +++ b/result_server/utils/results_loader.py @@ -2,7 +2,13 @@ from math import ceil from utils.result_file import get_file_confidential_tags -from utils.result_records import load_result_json, load_visible_result_json +from utils.estimated_table_rows import build_estimated_table_columns, build_estimated_table_row +from utils.result_records import ( + extract_result_uuid, + format_result_timestamp, + load_result_json, + load_visible_result_json, +) from utils.result_table_rows import build_result_table_row @@ -138,71 +144,18 @@ def load_estimated_results_table( ): continue - current = result_data.get("current_system", {}) - future = result_data.get("future_system", {}) - estimate_meta = result_data.get("estimate_metadata", {}) - applicability = result_data.get("applicability", {}) - - timestamp = _format_timestamp_from_filename(filename) - estimate_uuid = estimate_meta.get("estimation_result_uuid") or _extract_uuid_from_filename(filename) - - rows.append({ - "timestamp": estimate_meta.get("estimation_result_timestamp") or timestamp, - "code": result_data.get("code", ""), - "exp": result_data.get("exp", ""), - "systemA_system": current.get("system", ""), - "systemA_fom": current.get("fom", ""), - "systemA_target_nodes": current.get("target_nodes", ""), - "systemA_scaling_method": current.get("scaling_method", ""), - "systemA_bench_system": current.get("benchmark", {}).get("system", ""), - "systemA_bench_fom": current.get("benchmark", {}).get("fom", ""), - "systemA_bench_nodes": current.get("benchmark", {}).get("nodes", ""), - "systemB_system": future.get("system", ""), - "systemB_fom": future.get("fom", ""), - "systemB_target_nodes": future.get("target_nodes", ""), - "systemB_scaling_method": future.get("scaling_method", ""), - "systemB_bench_system": future.get("benchmark", {}).get("system", ""), - "systemB_bench_fom": future.get("benchmark", {}).get("fom", ""), - "systemB_bench_nodes": future.get("benchmark", {}).get("nodes", ""), - "applicability_status": applicability.get("status", ""), - "requested_estimation_package": estimate_meta.get("requested_estimation_package", ""), - "estimation_package": estimate_meta.get("estimation_package", ""), - "method_class": estimate_meta.get("method_class", ""), - "detail_level": estimate_meta.get("detail_level", ""), - "current_estimation_package": estimate_meta.get("current_package", {}).get("estimation_package", ""), - "future_estimation_package": estimate_meta.get("future_package", {}).get("estimation_package", ""), - "requested_current_estimation_package": estimate_meta.get("current_package", {}).get("requested_estimation_package", ""), - "requested_future_estimation_package": estimate_meta.get("future_package", {}).get("requested_estimation_package", ""), - "estimate_uuid": estimate_uuid or "", - "performance_ratio": result_data.get("performance_ratio", ""), - "json_link": filename, - }) - - columns = [ - ("Timestamp", "timestamp"), - ("CODE", "code"), - ("Exp", "exp"), - ("A System", "systemA_system"), - ("A FOM", "systemA_fom"), - ("A Target Nodes", "systemA_target_nodes"), - ("A Scaling Method", "systemA_scaling_method"), - ("A Bench System", "systemA_bench_system"), - ("A Bench FOM", "systemA_bench_fom"), - ("A Bench Nodes", "systemA_bench_nodes"), - ("B System", "systemB_system"), - ("B FOM", "systemB_fom"), - ("B Target Nodes", "systemB_target_nodes"), - ("B Scaling Method", "systemB_scaling_method"), - ("B Bench System", "systemB_bench_system"), - ("B Bench FOM", "systemB_bench_fom"), - ("B Bench Nodes", "systemB_bench_nodes"), - ("Applicability", "applicability_status"), - ("Requested Package", "requested_estimation_package"), - ("Applied Package", "estimation_package"), - ("Estimate UUID", "estimate_uuid"), - ("Performance Ratio", "performance_ratio"), - ("JSON", "json_link"), - ] + timestamp = format_result_timestamp(filename) + estimate_uuid = extract_result_uuid(filename) + rows.append( + build_estimated_table_row( + filename, + result_data, + fallback_uuid=estimate_uuid, + fallback_timestamp=timestamp, + ) + ) + + columns = build_estimated_table_columns() paginated_rows, pagination_info = paginate_list(rows, page, per_page) return paginated_rows, columns, pagination_info @@ -320,29 +273,3 @@ def _matches_table_filters(data, filter_system, filter_code, filter_exp, field_m if filter_exp is not None and _get_nested_field(data, field_map["exp"]) != filter_exp: return False return True - - -def _format_timestamp_from_filename(filename): - import re - from datetime import datetime - - match = re.search(r"\d{8}_\d{6}", filename) - if not match: - return "Unknown" - - try: - ts = datetime.strptime(match.group(), "%Y%m%d_%H%M%S") - except Exception: - return "Unknown" - return ts.strftime("%Y-%m-%d %H:%M:%S") - - -def _extract_uuid_from_filename(filename): - import re - - uuid_match = re.search( - r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", - filename, - re.IGNORECASE, - ) - return uuid_match.group(0) if uuid_match else None From bcef11b47efb347c06f3b57286fbb91c86c7a4e1 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 12:20:06 +0900 Subject: [PATCH 34/89] refactor: share portal table page helpers --- result_server/routes/estimated.py | 167 ++++++++++------ result_server/routes/results.py | 178 ++++++------------ .../templates/_estimated_results_table.html | 43 +++++ .../_estimated_results_table_cell.html | 23 +++ .../templates/estimated_results.html | 64 +------ result_server/tests/test_table_page_utils.py | 73 +++++++ .../tests/test_table_query_params.py | 44 +++++ result_server/utils/table_page_utils.py | 19 ++ result_server/utils/table_query_params.py | 20 ++ 9 files changed, 391 insertions(+), 240 deletions(-) create mode 100644 result_server/templates/_estimated_results_table.html create mode 100644 result_server/templates/_estimated_results_table_cell.html create mode 100644 result_server/tests/test_table_page_utils.py create mode 100644 result_server/tests/test_table_query_params.py create mode 100644 result_server/utils/table_page_utils.py create mode 100644 result_server/utils/table_query_params.py diff --git a/result_server/routes/estimated.py b/result_server/routes/estimated.py index 68fbeff..5200497 100644 --- a/result_server/routes/estimated.py +++ b/result_server/routes/estimated.py @@ -1,34 +1,98 @@ from flask import ( - Blueprint, render_template, request, session, - redirect, url_for, flash, abort, current_app, make_response + Blueprint, + abort, + current_app, + request, + redirect, + session, + url_for, ) + +from utils.result_file import check_file_permission, load_result_file from utils.result_records import load_result_json -from utils.results_loader import load_estimated_results_table, get_filter_options, ESTIMATED_FIELD_MAP -from routes.results import extract_query_params -from utils.user_store import get_user_store -from utils.result_file import load_result_file, check_file_permission +from utils.results_loader import ( + DEFAULT_PER_PAGE, + ESTIMATED_FIELD_MAP, + get_filter_options, + load_estimated_results_table, +) from utils.system_info import get_all_systems_info +from utils.table_page_utils import build_filtered_redirect_args, render_no_store_template +from utils.table_query_params import parse_table_query_params +from utils.user_store import get_user_store estimated_bp = Blueprint("estimated", __name__) +def _render_estimated_results_page(**context): + return render_no_store_template("estimated_results.html", **context) + + def _render_estimated_auth_required(): - systems_info = get_all_systems_info() - response = make_response(render_template( - "estimated_results.html", - rows=[], columns=[], - authenticated=False, systems_info=systems_info, - pagination={"page": 1, "per_page": 100, "total": 0, "total_pages": 1}, + return _render_estimated_results_page( + rows=[], + columns=[], + authenticated=False, + systems_info=get_all_systems_info(), + pagination={"page": 1, "per_page": DEFAULT_PER_PAGE, "total": 0, "total_pages": 1}, filter_options={"systems": [], "codes": [], "exps": []}, - current_system=None, current_code=None, - current_exp=None, current_per_page=100, - )) - response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" - response.headers["Pragma"] = "no-cache" - return response + current_system=None, + current_code=None, + current_exp=None, + current_per_page=DEFAULT_PER_PAGE, + ) + + +def _build_estimated_results_context( + estimated_dir, + email, + authenticated, + affiliations, + page, + per_page, + filter_system, + filter_code, + filter_exp, +): + rows, columns, pagination_info = load_estimated_results_table( + estimated_dir, + public_only=(not authenticated), + session_email=email, + authenticated=authenticated, + affiliations=affiliations, + page=page, + per_page=per_page, + filter_system=filter_system, + filter_code=filter_code, + filter_exp=filter_exp, + ) + + filter_options = get_filter_options( + estimated_dir, + public_only=(not authenticated), + authenticated=authenticated, + affiliations=affiliations, + field_map=ESTIMATED_FIELD_MAP, + ) + + return { + "rows": rows, + "columns": columns, + "authenticated": authenticated, + "systems_info": get_all_systems_info(), + "pagination": pagination_info, + "filter_options": filter_options, + "current_system": filter_system, + "current_code": filter_code, + "current_exp": filter_exp, + "current_per_page": per_page, + } + + +def _build_estimated_results_redirect_args(page, per_page, filter_system, filter_code, filter_exp): + return build_filtered_redirect_args(page, per_page, filter_system, filter_code, filter_exp) -# GET /estimated/ @estimated_bp.route("/", methods=["GET"], strict_slashes=False) def estimated_results(): authenticated = session.get("authenticated", False) @@ -36,12 +100,10 @@ def estimated_results(): return _render_estimated_auth_required() email = session.get("user_email") - store = get_user_store() - affs = store.get_affiliations(email) if email else [] + affiliations = store.get_affiliations(email) if email else [] - # クエリパラメータ取得 - params = extract_query_params() + params = parse_table_query_params(request.args) page = params["page"] per_page = params["per_page"] filter_system = params["filter_system"] @@ -49,49 +111,32 @@ def estimated_results(): filter_exp = params["filter_exp"] estimated_dir = current_app.config["ESTIMATED_DIR"] - - rows, columns, pagination_info = load_estimated_results_table( - estimated_dir, - public_only=(not authenticated), - session_email=email, + page_context = _build_estimated_results_context( + estimated_dir=estimated_dir, + email=email, authenticated=authenticated, - affiliations=affs, - page=page, per_page=per_page, - filter_system=filter_system, filter_code=filter_code, filter_exp=filter_exp, + affiliations=affiliations, + page=page, + per_page=per_page, + filter_system=filter_system, + filter_code=filter_code, + filter_exp=filter_exp, ) - # ページ範囲外の場合はリダイレクト + pagination_info = page_context["pagination"] if page != pagination_info["page"]: - redirect_args = {"page": pagination_info["page"], "per_page": per_page} - if filter_system is not None: - redirect_args["system"] = filter_system - if filter_code is not None: - redirect_args["code"] = filter_code - if filter_exp is not None: - redirect_args["exp"] = filter_exp + redirect_args = _build_estimated_results_redirect_args( + pagination_info["page"], + per_page, + filter_system, + filter_code, + filter_exp, + ) return redirect(url_for("estimated.estimated_results", **redirect_args)) - filter_options = get_filter_options( - estimated_dir, - public_only=(not authenticated), - authenticated=authenticated, affiliations=affs, - field_map=ESTIMATED_FIELD_MAP, - ) - systems_info = get_all_systems_info() - response = make_response(render_template( - "estimated_results.html", - rows=rows, columns=columns, - authenticated=authenticated, systems_info=systems_info, - pagination=pagination_info, filter_options=filter_options, - current_system=filter_system, current_code=filter_code, - current_exp=filter_exp, current_per_page=per_page, - )) - response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" - response.headers["Pragma"] = "no-cache" - return response - - -# GET /estimated/ + return _render_estimated_results_page(**page_context) + + @estimated_bp.route("/") def show_estimated_result(filename): if not session.get("authenticated", False): @@ -110,4 +155,6 @@ def estimated_detail(filename): result = load_result_json(filename, estimated_dir) if result is None: abort(404, "Estimated result file not found") + from flask import render_template + return render_template("estimated_detail.html", result=result, filename=filename) diff --git a/result_server/routes/results.py b/result_server/routes/results.py index 806df64..46e30e7 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -1,83 +1,56 @@ from datetime import datetime from flask import ( - Blueprint, render_template, request, session, - redirect, url_for, flash, abort, current_app, make_response + Blueprint, + abort, + current_app, + redirect, + render_template, + request, + session, + url_for, ) -from utils.result_records import load_result_json, load_result_json_batch, summarize_result_quality -from utils.results_loader import load_results_table, get_filter_options, ALLOWED_PER_PAGE, DEFAULT_PER_PAGE -from utils.user_store import get_user_store + +from routes.admin import admin_required from utils.app_support_matrix import load_app_system_support_matrix -from utils.site_diagnostics import build_site_diagnostics +from utils.node_hours import aggregate_node_hours, get_fiscal_year +from utils.result_file import check_file_permission, load_result_file from utils.result_quality_rollup import build_result_quality_rollup -from utils.result_file import load_result_file, check_file_permission +from utils.result_records import load_result_json, load_result_json_batch, summarize_result_quality +from utils.results_loader import DEFAULT_PER_PAGE, get_filter_options, load_results_table +from utils.site_diagnostics import build_site_diagnostics from utils.system_info import get_all_systems_info -from routes.admin import admin_required -from utils.node_hours import aggregate_node_hours, get_fiscal_year +from utils.table_page_utils import build_filtered_redirect_args, render_no_store_template +from utils.table_query_params import parse_table_query_params +from utils.user_store import get_user_store results_bp = Blueprint("results", __name__) def _render_confidential_auth_required(): systems_info = get_all_systems_info() - response = make_response(render_template( + return render_no_store_template( "results_confidential.html", - rows=[], columns=[], systems_info=systems_info, + rows=[], + columns=[], + systems_info=systems_info, pagination={"page": 1, "per_page": DEFAULT_PER_PAGE, "total": 0, "total_pages": 1}, filter_options={"systems": [], "codes": [], "exps": []}, - current_system=None, current_code=None, - current_exp=None, current_per_page=DEFAULT_PER_PAGE, + current_system=None, + current_code=None, + current_exp=None, + current_per_page=DEFAULT_PER_PAGE, authenticated=False, - )) - response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" - response.headers["Pragma"] = "no-cache" - return response - - -def extract_query_params(): - """request.args から page, per_page, system, code, exp を一括抽出する。 - - per_page が ALLOWED_PER_PAGE に含まれない場合は DEFAULT_PER_PAGE を使用。 - - Returns: - { - "page": int, - "per_page": int, - "filter_system": str | None, - "filter_code": str | None, - "filter_exp": str | None, - } - """ - page = request.args.get("page", 1, type=int) - per_page = request.args.get("per_page", 100, type=int) - filter_system = request.args.get("system", None) - filter_code = request.args.get("code", None) - filter_exp = request.args.get("exp", None) - - if per_page not in ALLOWED_PER_PAGE: - per_page = DEFAULT_PER_PAGE - - return { - "page": page, - "per_page": per_page, - "filter_system": filter_system, - "filter_code": filter_code, - "filter_exp": filter_exp, - } + ) def serve_confidential_file(filename, dir_path): - """ファイルアクセス権限確認して送信""" check_file_permission(filename, dir_path) return load_result_file(filename, dir_path) -def _render_results_list(public_only, template_name, redirect_endpoint): - """results() と results_confidential() の共通ロジック。 - クエリパラメータ抽出、セッション情報取得、データ読み込み、 - ページ範囲外リダイレクト、テンプレートレンダリングを一括処理する。 - """ - params = extract_query_params() +def _render_results_list(public_only, template_name, redirect_endpoint): + params = parse_table_query_params(request.args) page = params["page"] per_page = params["per_page"] filter_system = params["filter_system"] @@ -87,11 +60,13 @@ def _render_results_list(public_only, template_name, redirect_endpoint): received_dir = current_app.config["RECEIVED_DIR"] received_padata_dir = current_app.config["RECEIVED_PADATA_DIR"] - # public_only=False の場合のみセッション認証情報を取得 load_kwargs = dict( public_only=public_only, - page=page, per_page=per_page, - filter_system=filter_system, filter_code=filter_code, filter_exp=filter_exp, + page=page, + per_page=per_page, + filter_system=filter_system, + filter_code=filter_code, + filter_exp=filter_exp, padata_directory=received_padata_dir, ) filter_kwargs = dict(public_only=public_only) @@ -103,44 +78,42 @@ def _render_results_list(public_only, template_name, redirect_endpoint): return _render_confidential_auth_required() email = session.get("user_email") store = get_user_store() - affs = store.get_affiliations(email) if email else [] - load_kwargs.update(session_email=email, authenticated=authenticated, affiliations=affs) - filter_kwargs.update(authenticated=authenticated, affiliations=affs) + affiliations = store.get_affiliations(email) if email else [] + load_kwargs.update(session_email=email, authenticated=authenticated, affiliations=affiliations) + filter_kwargs.update(authenticated=authenticated, affiliations=affiliations) template_extra["authenticated"] = authenticated rows, columns, pagination_info = load_results_table(received_dir, **load_kwargs) - # ページ範囲外の場合はリダイレクト if page != pagination_info["page"]: - redirect_args = {"page": pagination_info["page"], "per_page": per_page} - if filter_system is not None: - redirect_args["system"] = filter_system - if filter_code is not None: - redirect_args["code"] = filter_code - if filter_exp is not None: - redirect_args["exp"] = filter_exp + redirect_args = build_filtered_redirect_args( + pagination_info["page"], + per_page, + filter_system, + filter_code, + filter_exp, + ) return redirect(url_for(redirect_endpoint, **redirect_args)) filter_options = get_filter_options(received_dir, filter_code=filter_code, **filter_kwargs) systems_info = get_all_systems_info() - response = make_response(render_template( - template_name, - rows=rows, columns=columns, systems_info=systems_info, - pagination=pagination_info, filter_options=filter_options, - current_system=filter_system, current_code=filter_code, - current_exp=filter_exp, current_per_page=per_page, + render_context = dict( + rows=rows, + columns=columns, + systems_info=systems_info, + pagination=pagination_info, + filter_options=filter_options, + current_system=filter_system, + current_code=filter_code, + current_exp=filter_exp, + current_per_page=per_page, **template_extra, - )) + ) if not public_only: - response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" - response.headers["Pragma"] = "no-cache" - return response + return render_no_store_template(template_name, **render_context) + return render_template(template_name, **render_context) -# ========================================== -# 公開用の結果一覧ページ -# GET /results/ -# ========================================== @results_bp.route("/", strict_slashes=False) def results(): return _render_results_list( @@ -150,11 +123,6 @@ def results(): ) -# ========================================== -# 機密データ付きの結果ページ（セッション認証） -# ========================================== - -# GET /results/confidential @results_bp.route("/confidential", methods=["GET"], strict_slashes=False) def results_confidential(): return _render_results_list( @@ -164,15 +132,10 @@ def results_confidential(): ) -# ========================================== -# リグレッション比較ページ -# GET /results/compare -# ========================================== @results_bp.route("/compare", methods=["GET"]) def result_compare(): - """複数結果のリグレッション比較ページ""" files_param = request.args.get("files", "") - filenames = [f.strip() for f in files_param.split(",") if f.strip()] + filenames = [name.strip() for name in files_param.split(",") if name.strip()] if len(filenames) < 2: abort(400, "Select 2 or more results to compare") @@ -186,21 +149,16 @@ def result_compare(): if results: first_system = results[0]["data"].get("system") first_code = results[0]["data"].get("code") - for r in results[1:]: - if r["data"].get("system") != first_system or r["data"].get("code") != first_code: + for row in results[1:]: + if row["data"].get("system") != first_system or row["data"].get("code") != first_code: mixed = True break return render_template("result_compare.html", results=results, mixed=mixed) -# ========================================== -# 個別結果の詳細ページ -# GET /results/detail/ -# ========================================== @results_bp.route("/detail/") def result_detail(filename): - """個別結果の詳細ページ（グラフ、データテーブル、ビルド情報）""" check_file_permission(filename, current_app.config["RECEIVED_DIR"]) result = load_result_json(filename, current_app.config["RECEIVED_DIR"]) if result is None: @@ -209,14 +167,9 @@ def result_detail(filename): return render_template("result_detail.html", result=result, filename=filename, quality=quality) -# ========================================== -# ノード時間使用量レポートページ -# GET /results/usage -# ========================================== @results_bp.route("/usage", methods=["GET"]) @admin_required def usage_report(): - """ノード時間使用量レポートページ""" valid_period_types = ("monthly", "semi_annual", "fiscal_year") period_type = request.args.get("period_type", "fiscal_year") if period_type not in valid_period_types: @@ -228,11 +181,8 @@ def usage_report(): except (TypeError, ValueError): fiscal_year = current_fy - result = aggregate_node_hours( - current_app.config["RECEIVED_DIR"], fiscal_year, period_type - ) + result = aggregate_node_hours(current_app.config["RECEIVED_DIR"], fiscal_year, period_type) - # 期間フィルタ: 月次→特定月、半期→上期/下期で絞り込み period_filter = request.args.get("period_filter", "") if period_filter and period_filter in result["periods"]: filtered_periods = [period_filter] @@ -263,15 +213,9 @@ def usage_report(): ) -# ========================================== -# 個別結果ファイルの表示/ダウンロード -# GET /results/ -# ========================================== @results_bp.route("/") def show_result(filename): if filename.endswith(".tgz"): - # TGZ payloads live in RECEIVED_PADATA_DIR, but confidentiality should - # still be resolved against the paired result JSON in RECEIVED_DIR. check_file_permission(filename, current_app.config["RECEIVED_DIR"]) return load_result_file(filename, current_app.config["RECEIVED_PADATA_DIR"]) return serve_confidential_file(filename, current_app.config["RECEIVED_DIR"]) diff --git a/result_server/templates/_estimated_results_table.html b/result_server/templates/_estimated_results_table.html new file mode 100644 index 0000000..3cc70bb --- /dev/null +++ b/result_server/templates/_estimated_results_table.html @@ -0,0 +1,43 @@ +{% set leading_columns = columns | selectattr("section", "equalto", "leading") | list %} +{% set trailing_columns = columns | selectattr("section", "equalto", "trailing") | list %} +{% set system_a_columns = columns | selectattr("group", "equalto", "System A") | list %} +{% set system_b_columns = columns | selectattr("group", "equalto", "System B") | list %} +
+
TimestampCodeExpSystem ASystem BApplicabilityRequested PackageApplied PackageEstimate UUIDRatioJSON{{ column.label }}System ASystem B{{ column.label }}
SystemFOMTarget NodesScaling MethodBench SystemBench FOMBench NodesSystemFOMTarget NodesScaling MethodBench SystemBench FOMBench Nodes{{ column.label }}{{ column.label }}
{{ ts_parts[0] }}
{{ ts_parts[1] if ts_parts|length > 1 else '' }}		{{ r.code }}{{ r.exp }}{{ r.systemA_system }}{{ a_fom }}{{ r.systemA_target_nodes }}{{ a_scaling_short }}{{ r.systemA_bench_system }}{{ "%.3f"|format(r.systemA_bench_fom|float) if r.systemA_bench_fom not in [None, '', 'N/A', 'null', 'nan'] else r.systemA_bench_fom }}{{ r.systemA_bench_nodes }}{{ r.systemB_system }}{{ b_fom }}{{ r.systemB_target_nodes }}{{ b_scaling_short }}{{ r.systemB_bench_system }}{{ "%.3f"|format(r.systemB_bench_fom|float) if r.systemB_bench_fom not in [None, '', 'N/A', 'null', 'nan'] else r.systemB_bench_fom }}{{ r.systemB_bench_nodes }}{{ r.applicability_status }}{{ requested_short }}{{ applied_short }}{% if r.method_class or r.detail_level %}
{{ r.method_class }}{% if r.detail_level %} / {{ r.detail_level }}{% endif %}
{% endif %}		{{ r.estimate_uuid[:8] if r.estimate_uuid else '' }}{{ ratio }} + {% for column in columns %} + {% set key = column.key %} + {% set title_value = r[column.title_key] if column.title_key is defined else None %} + {% set align_style = "text-align: " ~ column.align ~ ";" if column.align is defined else "" %} + {% if key == "timestamp" %} + {{ r.timestamp_date }}
{{ r.timestamp_time }}		 detail | json + {{ r[key] }} + {% if column.meta_key is defined and r[column.meta_key] %} +
{{ r[column.meta_key] }}
+ {% endif %} +
+ + + {% for column in leading_columns %} + + {% endfor %} + {% if system_a_columns %} + + {% endif %} + {% if system_b_columns %} + + {% endif %} + {% for column in trailing_columns %} + + {% endfor %} + + + {% for column in system_a_columns %} + + {% endfor %} + {% for column in system_b_columns %} + + {% endfor %} + + + + {% for row in rows %} + + {% for column in columns %} + {% with row=row, column=column %} + {% include "_estimated_results_table_cell.html" %} + {% endwith %} + {% endfor %} + + {% endfor %} + +
{{ column.label }}System ASystem B{{ column.label }}
{{ column.label }}{{ column.label }}
+
diff --git a/result_server/templates/_estimated_results_table_cell.html b/result_server/templates/_estimated_results_table_cell.html new file mode 100644 index 0000000..bf91368 --- /dev/null +++ b/result_server/templates/_estimated_results_table_cell.html @@ -0,0 +1,23 @@ +{% set key = column.key %} +{% set title_value = row[column.title_key] if column.title_key is defined else None %} +{% set align_style = "text-align: " ~ column.align ~ ";" if column.align is defined else "" %} +{% if key == "timestamp" %} +		{{ row.timestamp_date }}
{{ row.timestamp_time }}		 + detail + | + json + + {{ row[key] }} + {% if column.meta_key is defined and row[column.meta_key] %} +
{{ row[column.meta_key] }}
+ {% endif %} +
- - - {% for column in leading_columns %} - - {% endfor %} - {% if system_a_columns %} - - {% endif %} - {% if system_b_columns %} - - {% endif %} - {% for column in trailing_columns %} - - {% endfor %} - - - {% for column in system_a_columns %} - - {% endfor %} - {% for column in system_b_columns %} - - {% endfor %} - - - - {% for r in rows %} - - {% for column in columns %} - {% set key = column.key %} - {% set title_value = r[column.title_key] if column.title_key is defined else None %} - {% set align_style = "text-align: " ~ column.align ~ ";" if column.align is defined else "" %} - {% if key == "timestamp" %} - - {% elif key == "json_link" %} - - {% else %} - - {% endif %} - {% endfor %} - - {% endfor %} - -
{{ column.label }}System ASystem B{{ column.label }}
{{ column.label }}{{ column.label }}
{{ r.timestamp_date }}
{{ r.timestamp_time }}		 - detail - | - json - - {{ r[key] }} - {% if column.meta_key is defined and r[column.meta_key] %} -
{{ r[column.meta_key] }}
- {% endif %} -
-
+{% include "_estimated_results_table.html" %} {% include "_pagination.html" %} diff --git a/result_server/tests/test_table_page_utils.py b/result_server/tests/test_table_page_utils.py new file mode 100644 index 0000000..064e673 --- /dev/null +++ b/result_server/tests/test_table_page_utils.py @@ -0,0 +1,73 @@ +import os +import sys + +from flask import Blueprint, Flask + + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from utils.table_page_utils import build_filtered_redirect_args, render_no_store_template + + +def test_build_filtered_redirect_args_omits_missing_filters(): + args = build_filtered_redirect_args(2, 50, "Fugaku", None, "CASE0") + + assert args == { + "page": 2, + "per_page": 50, + "system": "Fugaku", + "exp": "CASE0", + } + + +def test_render_no_store_template_sets_cache_headers(): + app = Flask(__name__, template_folder=os.path.join(os.path.dirname(__file__), "..", "templates")) + app.config["SECRET_KEY"] = "test-secret" + + results_bp = Blueprint("results", __name__) + estimated_bp = Blueprint("estimated", __name__) + auth_bp = Blueprint("auth", __name__) + admin_bp = Blueprint("admin", __name__) + + @app.route("/") + def home(): + return "" + + @app.route("/systemlist") + def systemlist(): + return "" + + @results_bp.route("/") + def results(): + return "" + + @results_bp.route("/compare") + def result_compare(): + return "" + + @results_bp.route("/usage") + def usage_report(): + return "" + + @estimated_bp.route("/") + def estimated_results(): + return "" + + @auth_bp.route("/login") + def login(): + return "" + + @admin_bp.route("/users") + def users(): + return "" + + app.register_blueprint(results_bp, url_prefix="/results") + app.register_blueprint(estimated_bp, url_prefix="/estimated") + app.register_blueprint(auth_bp, url_prefix="/auth") + app.register_blueprint(admin_bp, url_prefix="/admin") + + with app.test_request_context("/estimated"): + response = render_no_store_template("estimated_results.html", authenticated=False, rows=[], columns=[], systems_info={}, pagination={"page": 1, "per_page": 100, "total": 0, "total_pages": 1}, filter_options={"systems": [], "codes": [], "exps": []}, current_system=None, current_code=None, current_exp=None, current_per_page=100) + + assert "no-store" in response.headers["Cache-Control"] + assert response.headers["Pragma"] == "no-cache" diff --git a/result_server/tests/test_table_query_params.py b/result_server/tests/test_table_query_params.py new file mode 100644 index 0000000..af0b5bb --- /dev/null +++ b/result_server/tests/test_table_query_params.py @@ -0,0 +1,44 @@ +import os +import sys + +from werkzeug.datastructures import MultiDict + + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from utils.table_query_params import parse_table_query_params + + +def test_parse_table_query_params_reads_filters_and_pagination(): + args = MultiDict([ + ("page", "3"), + ("per_page", "50"), + ("system", "Fugaku"), + ("code", "qws"), + ("exp", "CASE0"), + ]) + + params = parse_table_query_params(args) + + assert params == { + "page": 3, + "per_page": 50, + "filter_system": "Fugaku", + "filter_code": "qws", + "filter_exp": "CASE0", + } + + +def test_parse_table_query_params_falls_back_for_invalid_per_page(): + args = MultiDict([ + ("page", "1"), + ("per_page", "75"), + ]) + + params = parse_table_query_params(args) + + assert params["page"] == 1 + assert params["per_page"] == 100 + assert params["filter_system"] is None + assert params["filter_code"] is None + assert params["filter_exp"] is None diff --git a/result_server/utils/table_page_utils.py b/result_server/utils/table_page_utils.py new file mode 100644 index 0000000..01e78f6 --- /dev/null +++ b/result_server/utils/table_page_utils.py @@ -0,0 +1,19 @@ +from flask import make_response, render_template + + +def render_no_store_template(template_name, **context): + response = make_response(render_template(template_name, **context)) + response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" + response.headers["Pragma"] = "no-cache" + return response + + +def build_filtered_redirect_args(page, per_page, filter_system, filter_code, filter_exp): + redirect_args = {"page": page, "per_page": per_page} + if filter_system is not None: + redirect_args["system"] = filter_system + if filter_code is not None: + redirect_args["code"] = filter_code + if filter_exp is not None: + redirect_args["exp"] = filter_exp + return redirect_args diff --git a/result_server/utils/table_query_params.py b/result_server/utils/table_query_params.py new file mode 100644 index 0000000..a70f396 --- /dev/null +++ b/result_server/utils/table_query_params.py @@ -0,0 +1,20 @@ +from utils.results_loader import ALLOWED_PER_PAGE, DEFAULT_PER_PAGE + + +def parse_table_query_params(args): + page = args.get("page", 1, type=int) + per_page = args.get("per_page", DEFAULT_PER_PAGE, type=int) + filter_system = args.get("system", None) + filter_code = args.get("code", None) + filter_exp = args.get("exp", None) + + if per_page not in ALLOWED_PER_PAGE: + per_page = DEFAULT_PER_PAGE + + return { + "page": page, + "per_page": per_page, + "filter_system": filter_system, + "filter_code": filter_code, + "filter_exp": filter_exp, + } From 0e9377fbdf62c577e6cba314f2317729cff9ed62 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 12:23:58 +0900 Subject: [PATCH 35/89] refactor: extract usage query helpers --- result_server/routes/results.py | 24 +++----- .../tests/test_usage_query_params.py | 55 +++++++++++++++++++ result_server/tests/test_usage_route.py | 4 +- result_server/utils/usage_query_params.py | 26 +++++++++ 4 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 result_server/tests/test_usage_query_params.py create mode 100644 result_server/utils/usage_query_params.py diff --git a/result_server/routes/results.py b/result_server/routes/results.py index 46e30e7..8ff1786 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -23,6 +23,7 @@ from utils.table_page_utils import build_filtered_redirect_args, render_no_store_template from utils.table_query_params import parse_table_query_params from utils.user_store import get_user_store +from utils.usage_query_params import parse_usage_query_params, select_usage_periods results_bp = Blueprint("results", __name__) @@ -58,7 +59,7 @@ def _render_results_list(public_only, template_name, redirect_endpoint): filter_exp = params["filter_exp"] received_dir = current_app.config["RECEIVED_DIR"] - received_padata_dir = current_app.config["RECEIVED_PADATA_DIR"] + received_padata_dir = current_app.config.get("RECEIVED_PADATA_DIR", received_dir) load_kwargs = dict( public_only=public_only, @@ -170,25 +171,14 @@ def result_detail(filename): @results_bp.route("/usage", methods=["GET"]) @admin_required def usage_report(): - valid_period_types = ("monthly", "semi_annual", "fiscal_year") - period_type = request.args.get("period_type", "fiscal_year") - if period_type not in valid_period_types: - period_type = "fiscal_year" - current_fy = get_fiscal_year(datetime.now()) - try: - fiscal_year = int(request.args.get("fiscal_year", current_fy)) - except (TypeError, ValueError): - fiscal_year = current_fy + params = parse_usage_query_params(request.args, current_fy) + period_type = params["period_type"] + fiscal_year = params["fiscal_year"] + period_filter = params["period_filter"] result = aggregate_node_hours(current_app.config["RECEIVED_DIR"], fiscal_year, period_type) - - period_filter = request.args.get("period_filter", "") - if period_filter and period_filter in result["periods"]: - filtered_periods = [period_filter] - else: - period_filter = "" - filtered_periods = result["periods"] + period_filter, filtered_periods = select_usage_periods(result["periods"], period_filter) systems_info = get_all_systems_info() coverage_systems, app_support_rows = load_app_system_support_matrix() diff --git a/result_server/tests/test_usage_query_params.py b/result_server/tests/test_usage_query_params.py new file mode 100644 index 0000000..a5a1ed7 --- /dev/null +++ b/result_server/tests/test_usage_query_params.py @@ -0,0 +1,55 @@ +import os +import sys + +from werkzeug.datastructures import MultiDict + + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from utils.usage_query_params import parse_usage_query_params, select_usage_periods + + +def test_parse_usage_query_params_reads_usage_filters(): + args = MultiDict([ + ("period_type", "semi_annual"), + ("fiscal_year", "2026"), + ("period_filter", "H1"), + ]) + + params = parse_usage_query_params(args, 2025) + + assert params == { + "period_type": "semi_annual", + "fiscal_year": 2026, + "period_filter": "H1", + } + + +def test_parse_usage_query_params_falls_back_for_invalid_values(): + args = MultiDict([ + ("period_type", "weird"), + ("fiscal_year", "oops"), + ("period_filter", "FY2025"), + ]) + + params = parse_usage_query_params(args, 2025) + + assert params == { + "period_type": "fiscal_year", + "fiscal_year": 2025, + "period_filter": "FY2025", + } + + +def test_select_usage_periods_returns_single_period_when_present(): + period_filter, filtered_periods = select_usage_periods(["H1", "H2"], "H2") + + assert period_filter == "H2" + assert filtered_periods == ["H2"] + + +def test_select_usage_periods_resets_unknown_filter(): + period_filter, filtered_periods = select_usage_periods(["FY2025"], "H2") + + assert period_filter == "" + assert filtered_periods == ["FY2025"] diff --git a/result_server/tests/test_usage_route.py b/result_server/tests/test_usage_route.py index 6ecbc62..dfeceb2 100644 --- a/result_server/tests/test_usage_route.py +++ b/result_server/tests/test_usage_route.py @@ -174,8 +174,8 @@ def test_usage_page_shows_source_tracking_columns_when_rollup_exists(self, clien resp = client.get("/results/usage") text = resp.get_data(as_text=True) assert resp.status_code == 200 - assert "Source Status" in text - assert "Source Reference" in text + assert "Tracked" in text + assert "Reference" in text assert "top-level source tracked" in text def test_usage_route_uses_default_parameters(self, app, client, monkeypatch): diff --git a/result_server/utils/usage_query_params.py b/result_server/utils/usage_query_params.py new file mode 100644 index 0000000..1a0227d --- /dev/null +++ b/result_server/utils/usage_query_params.py @@ -0,0 +1,26 @@ +VALID_USAGE_PERIOD_TYPES = ("monthly", "semi_annual", "fiscal_year") + + +def parse_usage_query_params(args, current_fiscal_year): + period_type = args.get("period_type", "fiscal_year") + if period_type not in VALID_USAGE_PERIOD_TYPES: + period_type = "fiscal_year" + + try: + fiscal_year = int(args.get("fiscal_year", current_fiscal_year)) + except (TypeError, ValueError): + fiscal_year = current_fiscal_year + + period_filter = args.get("period_filter", "") + + return { + "period_type": period_type, + "fiscal_year": fiscal_year, + "period_filter": period_filter, + } + + +def select_usage_periods(periods, period_filter): + if period_filter and period_filter in periods: + return period_filter, [period_filter] + return "", periods From 1c47c4f8574b3a8b85c84441f75db8e9a26915c5 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 12:35:27 +0900 Subject: [PATCH 36/89] refactor: split usage report sections --- .../_usage_report_configuration_checks.html | 56 ++++ .../templates/_usage_report_controls.html | 37 +++ .../_usage_report_coverage_section.html | 43 +++ .../_usage_report_node_hours_section.html | 66 ++++ .../_usage_report_quality_section.html | 44 +++ result_server/templates/usage_report.html | 281 +----------------- 6 files changed, 262 insertions(+), 265 deletions(-) create mode 100644 result_server/templates/_usage_report_configuration_checks.html create mode 100644 result_server/templates/_usage_report_controls.html create mode 100644 result_server/templates/_usage_report_coverage_section.html create mode 100644 result_server/templates/_usage_report_node_hours_section.html create mode 100644 result_server/templates/_usage_report_quality_section.html diff --git a/result_server/templates/_usage_report_configuration_checks.html b/result_server/templates/_usage_report_configuration_checks.html new file mode 100644 index 0000000..726f883 --- /dev/null +++ b/result_server/templates/_usage_report_configuration_checks.html @@ -0,0 +1,56 @@ +
+

Configuration Checks

+

Lightweight checks for registered systems, queue definitions, and application support completeness.

+
+
+

Registered Systems

+

{{ site_diagnostics.registered_system_count }} systems are defined in system.csv.

+ {% if site_diagnostics.unused_systems %} +
    + {% for system in site_diagnostics.unused_systems %} +
  • {{ system }} has no fully enabled application support yet.
    • + {% endfor %} +
+ {% else %} +

All registered systems have at least one fully enabled application.

+ {% endif %} +
+
+

System Info Coverage

+ {% if site_diagnostics.missing_system_info %} +
    + {% for system in site_diagnostics.missing_system_info %} +
  • {{ system }} is missing a system_info.csv entry.
    • + {% endfor %} +
+ {% else %} +

Every registered system has a system_info.csv entry.

+ {% endif %} +
+
+

Queue Definitions

+ {% if site_diagnostics.missing_queue_definitions %} +
    + {% for item in site_diagnostics.missing_queue_definitions %} +
  • {{ item.system }} references queue {{ item.queue }} without a matching queue.csv definition.
    • + {% endfor %} +
+ {% else %} +

All queues referenced by system.csv are defined.

+ {% endif %} +
+
+

Partial Application Support

+

{{ site_diagnostics.application_count }} applications are scanned for registered systems.

+ {% if site_diagnostics.partial_support %} +
    + {% for item in site_diagnostics.partial_support %} +
  • {{ item.app }} / {{ item.system }} is enabled in list.csv, but script support is incomplete (build: {{ 'yes' if item.build_supported else 'no' }}, run: {{ 'yes' if item.run_supported else 'no' }}).
    • + {% endfor %} +
+ {% else %} +

No partial application/system support was detected.

+ {% endif %} +
+
+
diff --git a/result_server/templates/_usage_report_controls.html b/result_server/templates/_usage_report_controls.html new file mode 100644 index 0000000..a9967b7 --- /dev/null +++ b/result_server/templates/_usage_report_controls.html @@ -0,0 +1,37 @@ +
+ Period: + {% set buttons = [("monthly", "Monthly"), ("semi_annual", "Semi-Annual"), ("fiscal_year", "Fiscal Year")] %} + {% for val, label in buttons %} + + {{ label }} + + {% endfor %} + + Fiscal Year: + + + {% if period_type != 'fiscal_year' %} + Filter: + + {% endif %} + + +
diff --git a/result_server/templates/_usage_report_coverage_section.html b/result_server/templates/_usage_report_coverage_section.html new file mode 100644 index 0000000..283219e --- /dev/null +++ b/result_server/templates/_usage_report_coverage_section.html @@ -0,0 +1,43 @@ +
+

Application/System Coverage

+

Current support matrix based on list.csv plus detected build.sh / run.sh system branches.

+
+ ★ enabled and implemented + △ enabled in list.csv, script support incomplete + ○ configured off + — not listed +
+
+ + + + + {% for header in coverage_systems %} + + {% endfor %} + + + + {% for row in app_support_rows %} + + + {% for header in coverage_systems %} + {% set item = row.systems.get(header.system, {}) %} + {% if item.status == 'enabled' %} + + {% elif item.status == 'enabled_partial' %} + + {% elif item.status == 'configured_off' %} + + {% else %} + + {% endif %} + {% endfor %} + + {% else %} + + {% endfor %} + +
Application{{ header.name }}
{{ row.app }}★△○—
No application coverage data found.
+
+
diff --git a/result_server/templates/_usage_report_node_hours_section.html b/result_server/templates/_usage_report_node_hours_section.html new file mode 100644 index 0000000..72d6f3d --- /dev/null +++ b/result_server/templates/_usage_report_node_hours_section.html @@ -0,0 +1,66 @@ +
+

Node-Hour Usage

+

Aggregated node-hour usage across registered applications, systems, and the selected reporting periods.

+ + {% include "_usage_report_controls.html" %} + + {% if result.apps %} +
+ + + + + {% for system in result.systems %} + + {% endfor %} + + + + {% for system in result.systems %} + {% for period in filtered_periods %} + + {% endfor %} + {% endfor %} + {% for period in filtered_periods %} + + {% endfor %} + + + + {% for app in result.apps %} + + + {% for system in result.systems %} + {% for period in filtered_periods %} + {% set val = result.table[app][system][period] %} + + {% endfor %} + {% endfor %} + {% for period in filtered_periods %} + {% set val = result.row_totals[app][period] %} + + {% endfor %} + + {% endfor %} + + + + + {% for system in result.systems %} + {% for period in filtered_periods %} + {% set val = result.col_totals[system][period] %} + + {% endfor %} + {% endfor %} + {% for period in filtered_periods %} + {% set val = result.grand_totals[period] %} + + {% endfor %} + + +
{{ system }}Total
{{ period }}{{ period }}
{{ app }}{% if val > 0 %}{{ val }}{% else %}-{% endif %}{% if val > 0 %}{{ val }}{% else %}-{% endif %}
Total{% if val > 0 %}{{ val }}{% else %}-{% endif %}{% if val > 0 %}{{ val }}{% else %}-{% endif %}
+
+ {% else %} +

No usage data is available for the selected periods.

+ {% endif %} +
diff --git a/result_server/templates/_usage_report_quality_section.html b/result_server/templates/_usage_report_quality_section.html new file mode 100644 index 0000000..3ec5567 --- /dev/null +++ b/result_server/templates/_usage_report_quality_section.html @@ -0,0 +1,44 @@ +
+

Result Quality Coverage

+

Current-state check based on the latest collected result for each application/system pair, showing source provenance, breakdown output, and estimation-ready fields.

+ {% if result_quality_rollup.rows %} +
+ + + + + + + + + + + + + + + + + + {% for row in result_quality_rollup.rows %} + + + + + + + + + + + + + + {% endfor %} + +
ApplicationSystemLatest ResultTrackedTypeReferenceMissing FieldsBreakdownEst.-ReadyRichWarnings
{{ row.app }}{{ row.system }}{{ row.timestamp }}{{ row.source_status }}{{ row.source_type }}{{ row.source_reference }}{{ row.source_missing_fields|join(', ') if row.source_missing_fields else 'none' }}{{ 'yes' if row.breakdown_present else 'no' }}{{ 'yes' if row.estimation_ready else 'no' }}{{ 'yes' if row.rich else 'no' }}{{ row.warnings|join(', ') if row.warnings else 'none' }}
+
+ {% else %} +

No collected result-quality data is available yet.

+ {% endif %} +
diff --git a/result_server/templates/usage_report.html b/result_server/templates/usage_report.html index af8953e..c5fd273 100644 --- a/result_server/templates/usage_report.html +++ b/result_server/templates/usage_report.html @@ -4,8 +4,6 @@ {% block page_subtitle %}Review node-hour usage, registered coverage, configuration checks, and latest result current-state across applications and systems.{% endblock %} {% block content %} -
- - -
- 期間: - {% set buttons = [("monthly", "月次"), ("semi_annual", "半期"), ("fiscal_year", "年度")] %} - {% for val, label in buttons %} - - {{ label }} - - {% endfor %} - - - 年度: - - - {% if period_type != 'fiscal_year' %} - - 絞り込み: - - {% endif %} - - -
- - {% if result.apps %} - -
- - - - - - {% for system in result.systems %} - - {% endfor %} - - - - - {% for system in result.systems %} - {% for period in filtered_periods %} - - {% endfor %} - {% endfor %} - {% for period in filtered_periods %} - - {% endfor %} - - - - {% for app in result.apps %} - - - {% for system in result.systems %} - {% for period in filtered_periods %} - {% set val = result.table[app][system][period] %} - - {% endfor %} - {% endfor %} - {% for period in filtered_periods %} - {% set val = result.row_totals[app][period] %} - - {% endfor %} - - {% endfor %} - - - - - {% for system in result.systems %} - {% for period in filtered_periods %} - {% set val = result.col_totals[system][period] %} - - {% endfor %} - {% endfor %} - {% for period in filtered_periods %} - {% set val = result.grand_totals[period] %} - - {% endfor %} - - -
{{ system }}合計
{{ period }}{{ period }}
{{ app }}{% if val > 0 %}{{ val }}{% else %}-{% endif %}{% if val > 0 %}{{ val }}{% else %}-{% endif %}
合計{% if val > 0 %}{{ val }}{% else %}-{% endif %}{% if val > 0 %}{{ val }}{% else %}-{% endif %}
-
- {% else %} -

該当期間のデータがありません

- {% endif %} - -
- -
-

Configuration Checks

-

Lightweight checks for registered systems, queue definitions, and application support completeness.

-
-
-

Registered Systems

-

{{ site_diagnostics.registered_system_count }} systems are defined in system.csv.

- {% if site_diagnostics.unused_systems %} -
    - {% for system in site_diagnostics.unused_systems %} -
  • {{ system }} has no fully enabled application support yet.
    • - {% endfor %} -
- {% else %} -

All registered systems have at least one fully enabled application.

- {% endif %} -
-
-

System Info Coverage

- {% if site_diagnostics.missing_system_info %} -
    - {% for system in site_diagnostics.missing_system_info %} -
  • {{ system }} is missing a system_info.csv entry.
    • - {% endfor %} -
- {% else %} -

Every registered system has a system_info.csv entry.

- {% endif %} -
-
-

Queue Definitions

- {% if site_diagnostics.missing_queue_definitions %} -
    - {% for item in site_diagnostics.missing_queue_definitions %} -
  • {{ item.system }} references queue {{ item.queue }} without a matching queue.csv definition.
    • - {% endfor %} -
- {% else %} -

All queues referenced by system.csv are defined.

- {% endif %} -
-
-

Partial Application Support

-

{{ site_diagnostics.application_count }} applications are scanned for registered systems.

- {% if site_diagnostics.partial_support %} -
    - {% for item in site_diagnostics.partial_support %} -
  • {{ item.app }} / {{ item.system }} is enabled in list.csv, but script support is incomplete (build: {{ 'yes' if item.build_supported else 'no' }}, run: {{ 'yes' if item.run_supported else 'no' }}).
    • - {% endfor %} -
- {% else %} -

No partial application/system support was detected.

- {% endif %} -
-
-
- -
-

Application/System Coverage

-

Current support matrix based on list.csv plus detected build.sh / run.sh system branches.

-
- ★ enabled and implemented - △ enabled in list.csv, script support incomplete - ○ configured off - — not listed -
-
- - - - - {% for header in coverage_systems %} - - {% endfor %} - - - - {% for row in app_support_rows %} - - - {% for header in coverage_systems %} - {% set item = row.systems.get(header.system, {}) %} - {% if item.status == 'enabled' %} - - {% elif item.status == 'enabled_partial' %} - - {% elif item.status == 'configured_off' %} - - {% else %} - - {% endif %} - {% endfor %} - - {% else %} - - {% endfor %} - -
Application{{ header.name }}
{{ row.app }}★△○—
No application coverage data found.
-
-
- -
-

Result Quality Coverage

-

Current-state check based on the latest collected result for each application/system pair, showing source provenance, breakdown output, and estimation-ready fields.

- {% if result_quality_rollup.rows %} -
- - - - - - - - - - - - - - - - - - {% for row in result_quality_rollup.rows %} - - - - - - - - - - - - - - {% endfor %} - -
ApplicationSystemLatest ResultTrackedTypeReferenceMissing FieldsBreakdownEst.-ReadyRichWarnings
{{ row.app }}{{ row.system }}{{ row.timestamp }}{{ row.source_status }}{{ row.source_type }}{{ row.source_reference }}{{ row.source_missing_fields|join(', ') if row.source_missing_fields else 'none' }}{{ 'yes' if row.breakdown_present else 'no' }}{{ 'yes' if row.estimation_ready else 'no' }}{{ 'yes' if row.rich else 'no' }}{{ row.warnings|join(', ') if row.warnings else 'none' }}
-
- {% else %} -

No collected result-quality data is available yet.

- {% endif %} -
+ {% include "_usage_report_node_hours_section.html" %} + {% include "_usage_report_configuration_checks.html" %} + {% include "_usage_report_coverage_section.html" %} + {% include "_usage_report_quality_section.html" %} - + } + {% endblock %} From 006b2d380c6db2c3a7eabb2850f77a0d50747c11 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 12:48:03 +0900 Subject: [PATCH 37/89] refactor: compact estimated result table --- .../templates/estimated_results.html | 6 +- .../tests/test_estimated_results_loader.py | 12 +++ .../tests/test_portal_list_templates.py | 2 +- result_server/utils/estimated_table_rows.py | 79 ++++++++++--------- 4 files changed, 57 insertions(+), 42 deletions(-) diff --git a/result_server/templates/estimated_results.html b/result_server/templates/estimated_results.html index f9020d3..41d4e39 100644 --- a/result_server/templates/estimated_results.html +++ b/result_server/templates/estimated_results.html @@ -1,7 +1,7 @@ {% extends "_results_base.html" %} {% block title %}Estimated Results{% endblock %} -{% block page_subtitle %}Review projected performance and open detail for breakdowns or applicability. Estimated data requires authentication.{% endblock %} +{% block page_subtitle %}Review projected performance and open detail when you need breakdowns or applicability. Estimated data requires authentication.{% endblock %} {% block auth_warning %} {% if not authenticated %} @@ -32,7 +32,7 @@ border-radius: 14px; } .estimated-table { - min-width: 1780px; + min-width: 1640px; border-collapse: collapse; } .estimated-table td { @@ -53,7 +53,7 @@ } -

Focus on system pairs, applied packages, and ratio at the list level, then open detail when you need breakdowns or fallback/applicability context.

+

Scan system pairs, applied packages, and ratio here, then open detail for breakdown or fallback context.

{% include "_estimated_results_table.html" %} diff --git a/result_server/tests/test_estimated_results_loader.py b/result_server/tests/test_estimated_results_loader.py index be73c73..1ed2536 100644 --- a/result_server/tests/test_estimated_results_loader.py +++ b/result_server/tests/test_estimated_results_loader.py @@ -29,6 +29,7 @@ def _setup_stubs(): sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from flask import Flask +from utils.estimated_table_rows import build_estimated_table_columns from utils.results_loader import load_estimated_results_table @@ -106,3 +107,14 @@ def test_estimated_rows_prefer_metadata_fields(flask_app, tmp_dir): assert rows[0]["current_estimation_package"] == "weakscaling" assert rows[0]["future_estimation_package"] == "instrumented_app_sections_dummy" assert rows[0]["applicability_status"] == "fallback" + + +def test_estimated_columns_use_compact_labels(): + columns = build_estimated_table_columns() + labels = {column["key"]: column["label"] for column in columns} + + assert labels["systemA_target_nodes"] == "Nodes" + assert labels["systemA_scaling_short"] == "Scaling" + assert labels["requested_package_short"] == "Req. Pkg" + assert labels["applied_package_short"] == "Applied Pkg" + assert labels["estimate_uuid_short"] == "UUID" diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index 2884bf1..c465a9b 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -251,7 +251,7 @@ def test_estimated_results_template_renders_table_note(): filter_options={"systems": ["Fugaku"], "codes": ["qws"], "exps": ["CASE0"]}, ) - assert "Focus on system pairs, applied packages, and ratio" in html + assert "Scan system pairs, applied packages, and ratio here" in html assert "estimated-table-wrap" in html assert "detail" in html diff --git a/result_server/utils/estimated_table_rows.py b/result_server/utils/estimated_table_rows.py index a0af06c..ad538fd 100644 --- a/result_server/utils/estimated_table_rows.py +++ b/result_server/utils/estimated_table_rows.py @@ -16,32 +16,12 @@ def build_estimated_table_row(filename, result_data, fallback_uuid=None, fallbac current_package = estimate_meta.get("current_package", {}).get("estimation_package", "") future_package = estimate_meta.get("future_package", {}).get("estimation_package", "") - return { + row = { "timestamp": timestamp, "timestamp_date": _get_timestamp_date(timestamp), "timestamp_time": _get_timestamp_time(timestamp), "code": result_data.get("code", ""), "exp": result_data.get("exp", ""), - "systemA_system": current.get("system", ""), - "systemA_fom": current.get("fom", ""), - "systemA_fom_display": _format_numeric_display(current.get("fom", "")), - "systemA_target_nodes": current.get("target_nodes", ""), - "systemA_scaling_method": current.get("scaling_method", ""), - "systemA_scaling_title": current.get("scaling_method", ""), - "systemA_bench_system": current.get("benchmark", {}).get("system", ""), - "systemA_bench_fom": current.get("benchmark", {}).get("fom", ""), - "systemA_bench_fom_display": _format_numeric_display(current.get("benchmark", {}).get("fom", "")), - "systemA_bench_nodes": current.get("benchmark", {}).get("nodes", ""), - "systemB_system": future.get("system", ""), - "systemB_fom": future.get("fom", ""), - "systemB_fom_display": _format_numeric_display(future.get("fom", "")), - "systemB_target_nodes": future.get("target_nodes", ""), - "systemB_scaling_method": future.get("scaling_method", ""), - "systemB_scaling_title": future.get("scaling_method", ""), - "systemB_bench_system": future.get("benchmark", {}).get("system", ""), - "systemB_bench_fom": future.get("benchmark", {}).get("fom", ""), - "systemB_bench_fom_display": _format_numeric_display(future.get("benchmark", {}).get("fom", "")), - "systemB_bench_nodes": future.get("benchmark", {}).get("nodes", ""), "applicability_status": applicability.get("status", ""), "requested_estimation_package": requested_package, "estimation_package": applied_package, @@ -74,6 +54,9 @@ def build_estimated_table_row(filename, result_data, fallback_uuid=None, fallbac ), "applied_package_meta_line": _build_applied_package_meta_line(method_class, detail_level), } + row.update(_build_estimated_system_fields("systemA", current)) + row.update(_build_estimated_system_fields("systemB", future)) + return row def build_estimated_table_columns(): @@ -81,29 +64,49 @@ def build_estimated_table_columns(): {"label": "Timestamp", "key": "timestamp", "section": "leading"}, {"label": "CODE", "key": "code", "section": "leading"}, {"label": "Exp", "key": "exp", "section": "leading"}, - {"label": "System", "key": "systemA_system", "group": "System A"}, - {"label": "FOM", "key": "systemA_fom_display", "group": "System A", "title_key": "systemA_fom", "align": "right"}, - {"label": "Target Nodes", "key": "systemA_target_nodes", "group": "System A"}, - {"label": "Scaling Method", "key": "systemA_scaling_short", "group": "System A", "title_key": "systemA_scaling_title"}, - {"label": "Bench System", "key": "systemA_bench_system", "group": "System A"}, - {"label": "Bench FOM", "key": "systemA_bench_fom_display", "group": "System A", "title_key": "systemA_bench_fom", "align": "right"}, - {"label": "Bench Nodes", "key": "systemA_bench_nodes", "group": "System A"}, - {"label": "System", "key": "systemB_system", "group": "System B"}, - {"label": "FOM", "key": "systemB_fom_display", "group": "System B", "title_key": "systemB_fom", "align": "right"}, - {"label": "Target Nodes", "key": "systemB_target_nodes", "group": "System B"}, - {"label": "Scaling Method", "key": "systemB_scaling_short", "group": "System B", "title_key": "systemB_scaling_title"}, - {"label": "Bench System", "key": "systemB_bench_system", "group": "System B"}, - {"label": "Bench FOM", "key": "systemB_bench_fom_display", "group": "System B", "title_key": "systemB_bench_fom", "align": "right"}, - {"label": "Bench Nodes", "key": "systemB_bench_nodes", "group": "System B"}, + *_build_estimated_system_columns("System A", "systemA"), + *_build_estimated_system_columns("System B", "systemB"), {"label": "Applicability", "key": "applicability_status", "section": "trailing"}, - {"label": "Requested Package", "key": "requested_package_short", "section": "trailing", "title_key": "requested_package_title"}, - {"label": "Applied Package", "key": "applied_package_short", "section": "trailing", "title_key": "applied_package_title", "meta_key": "applied_package_meta_line"}, - {"label": "Estimate UUID", "key": "estimate_uuid_short", "section": "trailing", "title_key": "estimate_uuid", "cell_class": "estimated-code-cell"}, + {"label": "Req. Pkg", "key": "requested_package_short", "section": "trailing", "title_key": "requested_package_title"}, + {"label": "Applied Pkg", "key": "applied_package_short", "section": "trailing", "title_key": "applied_package_title", "meta_key": "applied_package_meta_line"}, + {"label": "UUID", "key": "estimate_uuid_short", "section": "trailing", "title_key": "estimate_uuid", "cell_class": "estimated-code-cell"}, {"label": "Ratio", "key": "performance_ratio_display", "section": "trailing", "title_key": "performance_ratio", "align": "right"}, {"label": "JSON", "key": "json_link", "section": "trailing", "cell_class": "estimated-link-cell"}, ] +def _build_estimated_system_fields(prefix, system_data): + benchmark = system_data.get("benchmark", {}) + fom = system_data.get("fom", "") + benchmark_fom = benchmark.get("fom", "") + scaling_method = system_data.get("scaling_method", "") + return { + f"{prefix}_system": system_data.get("system", ""), + f"{prefix}_fom": fom, + f"{prefix}_fom_display": _format_numeric_display(fom), + f"{prefix}_target_nodes": system_data.get("target_nodes", ""), + f"{prefix}_scaling_method": scaling_method, + f"{prefix}_scaling_title": scaling_method, + f"{prefix}_bench_system": benchmark.get("system", ""), + f"{prefix}_bench_fom": benchmark_fom, + f"{prefix}_bench_fom_display": _format_numeric_display(benchmark_fom), + f"{prefix}_bench_nodes": benchmark.get("nodes", ""), + f"{prefix}_scaling_short": _format_scaling_short_name(scaling_method), + } + + +def _build_estimated_system_columns(group_label, key_prefix): + return [ + {"label": "System", "key": f"{key_prefix}_system", "group": group_label}, + {"label": "FOM", "key": f"{key_prefix}_fom_display", "group": group_label, "title_key": f"{key_prefix}_fom", "align": "right"}, + {"label": "Nodes", "key": f"{key_prefix}_target_nodes", "group": group_label}, + {"label": "Scaling", "key": f"{key_prefix}_scaling_short", "group": group_label, "title_key": f"{key_prefix}_scaling_title"}, + {"label": "Bench", "key": f"{key_prefix}_bench_system", "group": group_label}, + {"label": "Bench FOM", "key": f"{key_prefix}_bench_fom_display", "group": group_label, "title_key": f"{key_prefix}_bench_fom", "align": "right"}, + {"label": "Bench Nodes", "key": f"{key_prefix}_bench_nodes", "group": group_label}, + ] + + def _format_scaling_short_name(value): if value == "instrumented-app-sections-dummy": return "instr-app-sec" From 5e433e39f81dae6825b9489b280f2a05e93f375c Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 13:03:03 +0900 Subject: [PATCH 38/89] refactor: move result detail display logic to view model --- result_server/routes/results.py | 4 +- result_server/templates/result_detail.html | 190 +++++------------- .../tests/test_result_detail_template.py | 89 ++++---- result_server/utils/result_detail_view.py | 142 +++++++++++++ 4 files changed, 224 insertions(+), 201 deletions(-) create mode 100644 result_server/utils/result_detail_view.py diff --git a/result_server/routes/results.py b/result_server/routes/results.py index 8ff1786..37d2127 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -14,6 +14,7 @@ from routes.admin import admin_required from utils.app_support_matrix import load_app_system_support_matrix from utils.node_hours import aggregate_node_hours, get_fiscal_year +from utils.result_detail_view import build_result_detail_context from utils.result_file import check_file_permission, load_result_file from utils.result_quality_rollup import build_result_quality_rollup from utils.result_records import load_result_json, load_result_json_batch, summarize_result_quality @@ -165,7 +166,8 @@ def result_detail(filename): if result is None: abort(404, "Result file not found") quality = summarize_result_quality(result) - return render_template("result_detail.html", result=result, filename=filename, quality=quality) + detail_context = build_result_detail_context(result, quality, filename) + return render_template("result_detail.html", result=result, quality=quality, **detail_context) @results_bp.route("/usage", methods=["GET"]) diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html index 0cf3a05..298fa08 100644 --- a/result_server/templates/result_detail.html +++ b/result_server/templates/result_detail.html @@ -2,7 +2,7 @@ - Result Detail - {{ result.get('Exp', 'Unknown') }} + Result Detail - {{ title_exp }} {% include "_table_base.html" %} @@ -25,17 +31,10 @@

Compare Results

Warning: Selected results have different system/code combinations. Comparison may not be meaningful.

{% endif %} -

{{ results[0]['data'].get('system', '') }} / {{ results[0]['data'].get('code', '') }} — Comparing {{ results | length }} results

- -{% set has_any_vector = [] %} -{% for r in results %} - {% if r['data'].get('metrics', {}).get('vector') %} - {% if has_any_vector.append(1) %}{% endif %} - {% endif %} -{% endfor %} +

{{ headline }}

-{% if has_any_vector %} +{% if has_vector_metrics %}

Vector Metrics - Overlay Graph

@@ -44,7 +43,6 @@

Vector Metrics - Overlay Graph

{% endif %} -

FOM Timeline

@@ -81,7 +79,10 @@

FOM Timeline

data: rows.map(function(row) { return {x: row[0], y: row[i]}; }), borderColor: color, backgroundColor: color.replace('1)', '0.1)'), - showLine: true, fill: false, tension: 0.1, pointRadius: 2 + showLine: true, + fill: false, + tension: 0.1, + pointRadius: 2 }); colorIdx++; } @@ -114,7 +115,9 @@

FOM Timeline

data: fomValues, borderColor: 'rgba(54, 162, 235, 1)', backgroundColor: 'rgba(54, 162, 235, 0.1)', - fill: true, tension: 0.1, pointRadius: 5 + fill: true, + tension: 0.1, + pointRadius: 5 }] }, options: { diff --git a/result_server/tests/test_portal_list_templates.py b/result_server/tests/test_portal_list_templates.py index c465a9b..a83f772 100644 --- a/result_server/tests/test_portal_list_templates.py +++ b/result_server/tests/test_portal_list_templates.py @@ -288,3 +288,31 @@ def test_usage_report_template_renders_search_box(): assert "Filter application/system coverage and current-state tables" in html assert "applyUsageSearch" in html + + +def test_result_compare_template_renders_headline(): + app = _make_app() + with app.test_request_context("/results/compare"): + from flask import render_template + + html = render_template( + "result_compare.html", + results=[ + { + "filename": "result0.json", + "timestamp": "2026-04-13 12:00:00", + "data": {"system": "Fugaku", "code": "qws", "FOM": 1.2}, + }, + { + "filename": "result1.json", + "timestamp": "2026-04-13 13:00:00", + "data": {"system": "Fugaku", "code": "qws", "FOM": 1.1}, + }, + ], + mixed=False, + headline="Fugaku / qws - Comparing 2 results", + has_vector_metrics=False, + ) + + assert "Fugaku / qws - Comparing 2 results" in html + assert "FOM Timeline" in html diff --git a/result_server/utils/result_compare_view.py b/result_server/utils/result_compare_view.py new file mode 100644 index 0000000..f55a210 --- /dev/null +++ b/result_server/utils/result_compare_view.py @@ -0,0 +1,16 @@ +def build_result_compare_context(results): + has_vector_metrics = any( + (row.get("data") or {}).get("metrics", {}).get("vector") + for row in results + ) + + headline = "" + if results: + first = results[0].get("data", {}) + headline = f"{first.get('system', '')} / {first.get('code', '')} - Comparing {len(results)} results" + + return { + "results": results, + "headline": headline, + "has_vector_metrics": has_vector_metrics, + } From 8c2828df842c49f318d13161fe25b09d3aa19352 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 13:16:58 +0900 Subject: [PATCH 40/89] refactor: share route table page builders --- result_server/routes/estimated.py | 47 ++++++++++---------- result_server/routes/results.py | 23 +++++----- result_server/tests/test_table_page_utils.py | 46 ++++++++++++++++++- result_server/utils/table_page_utils.py | 36 ++++++++++++++- 4 files changed, 115 insertions(+), 37 deletions(-) diff --git a/result_server/routes/estimated.py b/result_server/routes/estimated.py index 5200497..0d12130 100644 --- a/result_server/routes/estimated.py +++ b/result_server/routes/estimated.py @@ -3,9 +3,7 @@ abort, current_app, request, - redirect, session, - url_for, ) from utils.result_file import check_file_permission, load_result_file @@ -17,7 +15,11 @@ load_estimated_results_table, ) from utils.system_info import get_all_systems_info -from utils.table_page_utils import build_filtered_redirect_args, render_no_store_template +from utils.table_page_utils import ( + build_table_page_context, + build_table_page_redirect, + render_no_store_template, +) from utils.table_query_params import parse_table_query_params from utils.user_store import get_user_store @@ -29,18 +31,19 @@ def _render_estimated_results_page(**context): def _render_estimated_auth_required(): - return _render_estimated_results_page( + auth_required_context = build_table_page_context( rows=[], columns=[], - authenticated=False, - systems_info=get_all_systems_info(), pagination={"page": 1, "per_page": DEFAULT_PER_PAGE, "total": 0, "total_pages": 1}, filter_options={"systems": [], "codes": [], "exps": []}, current_system=None, current_code=None, current_exp=None, current_per_page=DEFAULT_PER_PAGE, + authenticated=False, + systems_info=get_all_systems_info(), ) + return _render_estimated_results_page(**auth_required_context) def _build_estimated_results_context( @@ -75,22 +78,18 @@ def _build_estimated_results_context( field_map=ESTIMATED_FIELD_MAP, ) - return { - "rows": rows, - "columns": columns, - "authenticated": authenticated, - "systems_info": get_all_systems_info(), - "pagination": pagination_info, - "filter_options": filter_options, - "current_system": filter_system, - "current_code": filter_code, - "current_exp": filter_exp, - "current_per_page": per_page, - } - - -def _build_estimated_results_redirect_args(page, per_page, filter_system, filter_code, filter_exp): - return build_filtered_redirect_args(page, per_page, filter_system, filter_code, filter_exp) + return build_table_page_context( + rows=rows, + columns=columns, + pagination=pagination_info, + filter_options=filter_options, + current_system=filter_system, + current_code=filter_code, + current_exp=filter_exp, + current_per_page=per_page, + authenticated=authenticated, + systems_info=get_all_systems_info(), + ) @estimated_bp.route("/", methods=["GET"], strict_slashes=False) @@ -125,14 +124,14 @@ def estimated_results(): pagination_info = page_context["pagination"] if page != pagination_info["page"]: - redirect_args = _build_estimated_results_redirect_args( + return build_table_page_redirect( + "estimated.estimated_results", pagination_info["page"], per_page, filter_system, filter_code, filter_exp, ) - return redirect(url_for("estimated.estimated_results", **redirect_args)) return _render_estimated_results_page(**page_context) diff --git a/result_server/routes/results.py b/result_server/routes/results.py index eabba70..76276d5 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -4,11 +4,9 @@ Blueprint, abort, current_app, - redirect, render_template, request, session, - url_for, ) from routes.admin import admin_required @@ -22,7 +20,11 @@ from utils.results_loader import DEFAULT_PER_PAGE, get_filter_options, load_results_table from utils.site_diagnostics import build_site_diagnostics from utils.system_info import get_all_systems_info -from utils.table_page_utils import build_filtered_redirect_args, render_no_store_template +from utils.table_page_utils import ( + build_table_page_context, + build_table_page_redirect, + render_no_store_template, +) from utils.table_query_params import parse_table_query_params from utils.user_store import get_user_store from utils.usage_query_params import parse_usage_query_params, select_usage_periods @@ -32,19 +34,19 @@ def _render_confidential_auth_required(): systems_info = get_all_systems_info() - return render_no_store_template( - "results_confidential.html", + auth_required_context = build_table_page_context( rows=[], columns=[], - systems_info=systems_info, pagination={"page": 1, "per_page": DEFAULT_PER_PAGE, "total": 0, "total_pages": 1}, filter_options={"systems": [], "codes": [], "exps": []}, current_system=None, current_code=None, current_exp=None, current_per_page=DEFAULT_PER_PAGE, + systems_info=systems_info, authenticated=False, ) + return render_no_store_template("results_confidential.html", **auth_required_context) def serve_confidential_file(filename, dir_path): @@ -89,27 +91,26 @@ def _render_results_list(public_only, template_name, redirect_endpoint): rows, columns, pagination_info = load_results_table(received_dir, **load_kwargs) if page != pagination_info["page"]: - redirect_args = build_filtered_redirect_args( + return build_table_page_redirect( + redirect_endpoint, pagination_info["page"], per_page, filter_system, filter_code, filter_exp, ) - return redirect(url_for(redirect_endpoint, **redirect_args)) filter_options = get_filter_options(received_dir, filter_code=filter_code, **filter_kwargs) - systems_info = get_all_systems_info() - render_context = dict( + render_context = build_table_page_context( rows=rows, columns=columns, - systems_info=systems_info, pagination=pagination_info, filter_options=filter_options, current_system=filter_system, current_code=filter_code, current_exp=filter_exp, current_per_page=per_page, + systems_info=get_all_systems_info(), **template_extra, ) if not public_only: diff --git a/result_server/tests/test_table_page_utils.py b/result_server/tests/test_table_page_utils.py index 064e673..c76d8ec 100644 --- a/result_server/tests/test_table_page_utils.py +++ b/result_server/tests/test_table_page_utils.py @@ -6,7 +6,12 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from utils.table_page_utils import build_filtered_redirect_args, render_no_store_template +from utils.table_page_utils import ( + build_filtered_redirect_args, + build_table_page_context, + build_table_page_redirect, + render_no_store_template, +) def test_build_filtered_redirect_args_omits_missing_filters(): @@ -71,3 +76,42 @@ def users(): assert "no-store" in response.headers["Cache-Control"] assert response.headers["Pragma"] == "no-cache" + + +def test_build_table_page_context_keeps_common_keys(): + context = build_table_page_context( + rows=[{"filename": "result0.json"}], + columns=[{"key": "system", "label": "System"}], + pagination={"page": 2, "per_page": 50, "total": 3, "total_pages": 1}, + filter_options={"systems": ["Fugaku"], "codes": ["qws"], "exps": ["CASE0"]}, + current_system="Fugaku", + current_code="qws", + current_exp="CASE0", + current_per_page=50, + systems_info={"Fugaku": {"name": "Fugaku"}}, + authenticated=True, + ) + + assert context["rows"] == [{"filename": "result0.json"}] + assert context["columns"][0]["key"] == "system" + assert context["pagination"]["page"] == 2 + assert context["systems_info"]["Fugaku"]["name"] == "Fugaku" + assert context["authenticated"] is True + + +def test_build_table_page_redirect_uses_filtered_args(): + app = Flask(__name__) + + results_bp = Blueprint("results", __name__) + + @results_bp.route("/") + def results(): + return "" + + app.register_blueprint(results_bp, url_prefix="/results") + + with app.test_request_context("/results"): + response = build_table_page_redirect("results.results", 3, 20, "Fugaku", "qws", None) + + assert response.status_code == 302 + assert response.location.endswith("/results/?page=3&per_page=20&system=Fugaku&code=qws") diff --git a/result_server/utils/table_page_utils.py b/result_server/utils/table_page_utils.py index 01e78f6..1e0101a 100644 --- a/result_server/utils/table_page_utils.py +++ b/result_server/utils/table_page_utils.py @@ -1,4 +1,4 @@ -from flask import make_response, render_template +from flask import make_response, redirect, render_template, url_for def render_no_store_template(template_name, **context): @@ -17,3 +17,37 @@ def build_filtered_redirect_args(page, per_page, filter_system, filter_code, fil if filter_exp is not None: redirect_args["exp"] = filter_exp return redirect_args + + +def build_table_page_context( + *, + rows, + columns, + pagination, + filter_options, + current_system, + current_code, + current_exp, + current_per_page, + systems_info=None, + **extra_context, +): + context = { + "rows": rows, + "columns": columns, + "pagination": pagination, + "filter_options": filter_options, + "current_system": current_system, + "current_code": current_code, + "current_exp": current_exp, + "current_per_page": current_per_page, + } + if systems_info is not None: + context["systems_info"] = systems_info + context.update(extra_context) + return context + + +def build_table_page_redirect(endpoint, page, per_page, filter_system, filter_code, filter_exp): + redirect_args = build_filtered_redirect_args(page, per_page, filter_system, filter_code, filter_exp) + return redirect(url_for(endpoint, **redirect_args)) From 5b73df31df864dd6a58a24dc2b18ccca66c178f1 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 13:21:39 +0900 Subject: [PATCH 41/89] refactor: share result file access helpers --- result_server/routes/estimated.py | 26 ++++---- result_server/routes/results.py | 27 +++++---- result_server/utils/result_file.py | 96 ++++++++++++++++++++---------- 3 files changed, 95 insertions(+), 54 deletions(-) diff --git a/result_server/routes/estimated.py b/result_server/routes/estimated.py index 0d12130..945977d 100644 --- a/result_server/routes/estimated.py +++ b/result_server/routes/estimated.py @@ -1,13 +1,15 @@ from flask import ( Blueprint, - abort, current_app, request, session, ) -from utils.result_file import check_file_permission, load_result_file -from utils.result_records import load_result_json +from utils.result_file import ( + load_permitted_result_json, + require_authenticated_session, + serve_permitted_result_file, +) from utils.results_loader import ( DEFAULT_PER_PAGE, ESTIMATED_FIELD_MAP, @@ -138,22 +140,20 @@ def estimated_results(): @estimated_bp.route("/") def show_estimated_result(filename): - if not session.get("authenticated", False): - abort(403, "Authentication required to view estimated data") + require_authenticated_session("Authentication required to view estimated data") estimated_dir = current_app.config["ESTIMATED_DIR"] - check_file_permission(filename, estimated_dir) - return load_result_file(filename, estimated_dir) + return serve_permitted_result_file(filename, estimated_dir) @estimated_bp.route("/detail/") def estimated_detail(filename): - if not session.get("authenticated", False): - abort(403, "Authentication required to view estimated data") + require_authenticated_session("Authentication required to view estimated data") estimated_dir = current_app.config["ESTIMATED_DIR"] - check_file_permission(filename, estimated_dir) - result = load_result_json(filename, estimated_dir) - if result is None: - abort(404, "Estimated result file not found") + result = load_permitted_result_json( + filename, + estimated_dir, + not_found_message="Estimated result file not found", + ) from flask import render_template return render_template("estimated_detail.html", result=result, filename=filename) diff --git a/result_server/routes/results.py b/result_server/routes/results.py index 76276d5..499614b 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -14,9 +14,13 @@ from utils.node_hours import aggregate_node_hours, get_fiscal_year from utils.result_compare_view import build_result_compare_context from utils.result_detail_view import build_result_detail_context -from utils.result_file import check_file_permission, load_result_file +from utils.result_file import ( + check_file_permission, + load_permitted_result_json, + serve_permitted_result_file, +) from utils.result_quality_rollup import build_result_quality_rollup -from utils.result_records import load_result_json, load_result_json_batch, summarize_result_quality +from utils.result_records import load_result_json_batch, summarize_result_quality from utils.results_loader import DEFAULT_PER_PAGE, get_filter_options, load_results_table from utils.site_diagnostics import build_site_diagnostics from utils.system_info import get_all_systems_info @@ -50,8 +54,7 @@ def _render_confidential_auth_required(): def serve_confidential_file(filename, dir_path): - check_file_permission(filename, dir_path) - return load_result_file(filename, dir_path) + return serve_permitted_result_file(filename, dir_path) def _render_results_list(public_only, template_name, redirect_endpoint): @@ -164,10 +167,11 @@ def result_compare(): @results_bp.route("/detail/") def result_detail(filename): - check_file_permission(filename, current_app.config["RECEIVED_DIR"]) - result = load_result_json(filename, current_app.config["RECEIVED_DIR"]) - if result is None: - abort(404, "Result file not found") + result = load_permitted_result_json( + filename, + current_app.config["RECEIVED_DIR"], + not_found_message="Result file not found", + ) quality = summarize_result_quality(result) detail_context = build_result_detail_context(result, quality, filename) return render_template("result_detail.html", result=result, quality=quality, **detail_context) @@ -211,6 +215,9 @@ def usage_report(): @results_bp.route("/") def show_result(filename): if filename.endswith(".tgz"): - check_file_permission(filename, current_app.config["RECEIVED_DIR"]) - return load_result_file(filename, current_app.config["RECEIVED_PADATA_DIR"]) + return serve_permitted_result_file( + filename, + current_app.config["RECEIVED_DIR"], + current_app.config["RECEIVED_PADATA_DIR"], + ) return serve_confidential_file(filename, current_app.config["RECEIVED_DIR"]) diff --git a/result_server/utils/result_file.py b/result_server/utils/result_file.py index 5aafe51..7468809 100644 --- a/result_server/utils/result_file.py +++ b/result_server/utils/result_file.py @@ -1,9 +1,12 @@ -import os import json +import os import re -from flask import abort, jsonify, send_from_directory, session, Response + +from flask import Response, abort, send_from_directory, session + from utils.user_store import get_user_store + def load_result_file(filename: str, save_dir: str): filepath = os.path.join(save_dir, filename) if not os.path.exists(filepath): @@ -15,71 +18,102 @@ def load_result_file(filename: str, save_dir: str): data = json.load(f) return Response( json.dumps(data, indent=4, ensure_ascii=False), - mimetype="application/json" + mimetype="application/json", ) except json.JSONDecodeError: abort(400, "Invalid JSON") - else: - abs_dir = os.path.abspath(save_dir) - return send_from_directory(abs_dir, filename, as_attachment=True) + + abs_dir = os.path.abspath(save_dir) + return send_from_directory(abs_dir, filename, as_attachment=True) def get_file_confidential_tags(filename: str, save_dir: str): - """ - JSON/TGZのconfidentialタグ取得 - """ + """Return confidential tags from a JSON file or its matching PA archive.""" if filename.endswith(".json"): return _read_confidential_from_json(filename, save_dir) - # tgzの場合、対応するUUIDを含むJSONを探す - uuid_match = re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", filename, re.IGNORECASE) + # For TGZ files, find the matching JSON by UUID and reuse its tags. + uuid_match = re.search( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + filename, + re.IGNORECASE, + ) if not uuid_match: return [] uuid = uuid_match.group(0) - for f in os.listdir(save_dir): - if f.endswith(".json") and uuid in f: - return _read_confidential_from_json(f, save_dir) + for json_filename in os.listdir(save_dir): + if json_filename.endswith(".json") and uuid in json_filename: + return _read_confidential_from_json(json_filename, save_dir) return [] def check_file_permission(filename: str, dir_path: str) -> None: - """ - ファイルの confidential タグを確認し、アクセス権限がなければ abort(403) する。 - 公開ファイル（タグなし）の場合は何もしない。 - """ + """Abort with 403 when the current session cannot access the file.""" tags = get_file_confidential_tags(filename, dir_path) if not tags: return + authenticated = session.get("authenticated", False) email = session.get("user_email") store = get_user_store() - affs = store.get_affiliations(email) if email else [] - if not authenticated or not (set(tags) & set(affs)): + affiliations = store.get_affiliations(email) if email else [] + if not authenticated or not (set(tags) & set(affiliations)): abort(403, "You do not have permission to access this file") +def require_authenticated_session(message: str) -> None: + """Abort with 403 when the current session is not authenticated.""" + if not session.get("authenticated", False): + abort(403, message) + + +def serve_permitted_result_file(filename: str, permission_dir: str, data_dir: str | None = None): + """Check permission tags and then serve the requested file.""" + check_file_permission(filename, permission_dir) + return load_result_file(filename, data_dir or permission_dir) + + +def load_permitted_result_json( + filename: str, + permission_dir: str, + data_dir: str | None = None, + *, + not_found_message: str = "Result file not found", +): + """Check permission tags and then load JSON content with a custom 404 message.""" + from utils.result_records import load_result_json + + check_file_permission(filename, permission_dir) + result = load_result_json(filename, data_dir or permission_dir) + if result is None: + abort(404, not_found_message) + return result + + def _read_confidential_from_json(json_file: str, save_dir: str): filepath = os.path.join(save_dir, json_file) if not os.path.exists(filepath): return [] + try: with open(filepath, "r", encoding="utf-8") as f: data = json.load(f) - # confidential キーがない場合は None と同じ扱いにする - c = data.get("confidential", None) + confidential_value = data.get("confidential", None) - if isinstance(c, list): - # null や空文字は除外 - return [str(x).strip() for x in c if x and str(x).lower() != "null"] - elif isinstance(c, str): - c = c.strip() - if c.lower() != "null" and c != "": - return [c] + if isinstance(confidential_value, list): + return [ + str(item).strip() + for item in confidential_value + if item and str(item).lower() != "null" + ] - # None または null の場合は空リスト - return [] + if isinstance(confidential_value, str): + confidential_value = confidential_value.strip() + if confidential_value.lower() != "null" and confidential_value != "": + return [confidential_value] + return [] except Exception: return [] From 045c1e923ae07331e88ec2b01f9f79e96ef13e70 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 13:24:15 +0900 Subject: [PATCH 42/89] fix: support Python 3.9 annotations in result file helpers --- result_server/utils/result_file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/result_server/utils/result_file.py b/result_server/utils/result_file.py index 7468809..1d82c0f 100644 --- a/result_server/utils/result_file.py +++ b/result_server/utils/result_file.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import os import re From a2194573c5b86e45e97c62f355b2a9c6f616c3d6 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 13:35:01 +0900 Subject: [PATCH 43/89] refactor: streamline portal session and compare helpers --- result_server/routes/estimated.py | 17 +-- result_server/routes/results.py | 46 +++----- .../tests/test_result_compare_view.py | 52 ++++++++ .../tests/test_session_user_context.py | 69 +++++++++++ result_server/utils/node_hours.py | 111 +++++------------- result_server/utils/result_compare_view.py | 23 ++++ result_server/utils/result_file.py | 9 +- result_server/utils/session_user_context.py | 17 +++ result_server/utils/user_store.py | 63 +++++----- 9 files changed, 251 insertions(+), 156 deletions(-) create mode 100644 result_server/tests/test_result_compare_view.py create mode 100644 result_server/tests/test_session_user_context.py create mode 100644 result_server/utils/session_user_context.py diff --git a/result_server/routes/estimated.py b/result_server/routes/estimated.py index 945977d..8e7c1d5 100644 --- a/result_server/routes/estimated.py +++ b/result_server/routes/estimated.py @@ -2,7 +2,6 @@ Blueprint, current_app, request, - session, ) from utils.result_file import ( @@ -16,6 +15,7 @@ get_filter_options, load_estimated_results_table, ) +from utils.session_user_context import get_session_user_context from utils.system_info import get_all_systems_info from utils.table_page_utils import ( build_table_page_context, @@ -23,7 +23,6 @@ render_no_store_template, ) from utils.table_query_params import parse_table_query_params -from utils.user_store import get_user_store estimated_bp = Blueprint("estimated", __name__) @@ -96,14 +95,10 @@ def _build_estimated_results_context( @estimated_bp.route("/", methods=["GET"], strict_slashes=False) def estimated_results(): - authenticated = session.get("authenticated", False) - if not authenticated: + user_context = get_session_user_context() + if not user_context["authenticated"]: return _render_estimated_auth_required() - email = session.get("user_email") - store = get_user_store() - affiliations = store.get_affiliations(email) if email else [] - params = parse_table_query_params(request.args) page = params["page"] per_page = params["per_page"] @@ -114,9 +109,9 @@ def estimated_results(): estimated_dir = current_app.config["ESTIMATED_DIR"] page_context = _build_estimated_results_context( estimated_dir=estimated_dir, - email=email, - authenticated=authenticated, - affiliations=affiliations, + email=user_context["email"], + authenticated=user_context["authenticated"], + affiliations=user_context["affiliations"], page=page, per_page=per_page, filter_system=filter_system, diff --git a/result_server/routes/results.py b/result_server/routes/results.py index 499614b..81f732e 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -6,22 +6,21 @@ current_app, render_template, request, - session, ) from routes.admin import admin_required from utils.app_support_matrix import load_app_system_support_matrix from utils.node_hours import aggregate_node_hours, get_fiscal_year -from utils.result_compare_view import build_result_compare_context +from utils.result_compare_view import load_result_compare_context from utils.result_detail_view import build_result_detail_context from utils.result_file import ( - check_file_permission, load_permitted_result_json, serve_permitted_result_file, ) from utils.result_quality_rollup import build_result_quality_rollup -from utils.result_records import load_result_json_batch, summarize_result_quality +from utils.result_records import summarize_result_quality from utils.results_loader import DEFAULT_PER_PAGE, get_filter_options, load_results_table +from utils.session_user_context import get_session_user_context from utils.site_diagnostics import build_site_diagnostics from utils.system_info import get_all_systems_info from utils.table_page_utils import ( @@ -30,7 +29,6 @@ render_no_store_template, ) from utils.table_query_params import parse_table_query_params -from utils.user_store import get_user_store from utils.usage_query_params import parse_usage_query_params, select_usage_periods results_bp = Blueprint("results", __name__) @@ -81,15 +79,19 @@ def _render_results_list(public_only, template_name, redirect_endpoint): template_extra = {} if not public_only: - authenticated = session.get("authenticated", False) - if not authenticated: + user_context = get_session_user_context() + if not user_context["authenticated"]: return _render_confidential_auth_required() - email = session.get("user_email") - store = get_user_store() - affiliations = store.get_affiliations(email) if email else [] - load_kwargs.update(session_email=email, authenticated=authenticated, affiliations=affiliations) - filter_kwargs.update(authenticated=authenticated, affiliations=affiliations) - template_extra["authenticated"] = authenticated + load_kwargs.update( + session_email=user_context["email"], + authenticated=user_context["authenticated"], + affiliations=user_context["affiliations"], + ) + filter_kwargs.update( + authenticated=user_context["authenticated"], + affiliations=user_context["affiliations"], + ) + template_extra["authenticated"] = user_context["authenticated"] rows, columns, pagination_info = load_results_table(received_dir, **load_kwargs) @@ -147,22 +149,8 @@ def result_compare(): if len(filenames) < 2: abort(400, "Select 2 or more results to compare") - for filename in filenames: - check_file_permission(filename, current_app.config["RECEIVED_DIR"]) - - results = load_result_json_batch(filenames, current_app.config["RECEIVED_DIR"]) - - mixed = False - if results: - first_system = results[0]["data"].get("system") - first_code = results[0]["data"].get("code") - for row in results[1:]: - if row["data"].get("system") != first_system or row["data"].get("code") != first_code: - mixed = True - break - - compare_context = build_result_compare_context(results) - return render_template("result_compare.html", mixed=mixed, **compare_context) + compare_context = load_result_compare_context(filenames, current_app.config["RECEIVED_DIR"]) + return render_template("result_compare.html", **compare_context) @results_bp.route("/detail/") diff --git a/result_server/tests/test_result_compare_view.py b/result_server/tests/test_result_compare_view.py new file mode 100644 index 0000000..b939c92 --- /dev/null +++ b/result_server/tests/test_result_compare_view.py @@ -0,0 +1,52 @@ +import os +import sys +import types + + +def _setup_stubs(): + if "redis" not in sys.modules: + sys.modules["redis"] = types.ModuleType("redis") + + otp_mod = types.ModuleType("utils.otp_manager") + otp_mod.get_affiliations = lambda email: ["dev"] + otp_mod.is_allowed = lambda email: True + sys.modules["utils.otp_manager"] = otp_mod + + otp_redis_mod = types.ModuleType("utils.otp_redis_manager") + otp_redis_mod.get_affiliations = lambda email: ["dev"] + otp_redis_mod.is_allowed = lambda email: True + otp_redis_mod.send_otp = lambda email: (True, "stub") + otp_redis_mod.verify_otp = lambda email, code: True + otp_redis_mod.invalidate_otp = lambda email: None + sys.modules["utils.otp_redis_manager"] = otp_redis_mod + + +_setup_stubs() + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from utils.result_compare_view import build_result_compare_context + + +def test_build_result_compare_context_marks_same_system_code_as_not_mixed(): + context = build_result_compare_context( + [ + {"data": {"system": "Fugaku", "code": "qws", "FOM": 1.0}}, + {"data": {"system": "Fugaku", "code": "qws", "FOM": 0.9}}, + ] + ) + + assert context["headline"] == "Fugaku / qws - Comparing 2 results" + assert context["mixed"] is False + assert context["has_vector_metrics"] is False + + +def test_build_result_compare_context_marks_mixed_rows(): + context = build_result_compare_context( + [ + {"data": {"system": "Fugaku", "code": "qws"}}, + {"data": {"system": "Other", "code": "qws"}}, + ] + ) + + assert context["mixed"] is True diff --git a/result_server/tests/test_session_user_context.py b/result_server/tests/test_session_user_context.py new file mode 100644 index 0000000..59aebff --- /dev/null +++ b/result_server/tests/test_session_user_context.py @@ -0,0 +1,69 @@ +import os +import sys +import types + +import pytest +from flask import Flask + + +def _setup_stubs(): + otp_mod = types.ModuleType("utils.otp_manager") + otp_mod.get_affiliations = lambda email: ["dev"] + otp_mod.is_allowed = lambda email: True + sys.modules["utils.otp_manager"] = otp_mod + + otp_redis_mod = types.ModuleType("utils.otp_redis_manager") + otp_redis_mod.get_affiliations = lambda email: ["dev"] + otp_redis_mod.is_allowed = lambda email: True + otp_redis_mod.send_otp = lambda email: (True, "stub") + otp_redis_mod.verify_otp = lambda email, code: True + otp_redis_mod.invalidate_otp = lambda email: None + sys.modules["utils.otp_redis_manager"] = otp_redis_mod + + +_setup_stubs() + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from utils.session_user_context import get_session_user_context + + +class _StubUserStore: + def get_affiliations(self, email): + if email == "user@example.com": + return ["dev"] + return [] + + +@pytest.fixture +def app(): + app = Flask(__name__) + app.config["SECRET_KEY"] = "test-secret" + app.config["TESTING"] = True + app.config["USER_STORE"] = _StubUserStore() + + return app + + +def test_get_session_user_context_for_anonymous_session(app): + with app.test_request_context("/results"): + context = get_session_user_context() + + assert context == { + "authenticated": False, + "email": None, + "affiliations": [], + } + + +def test_get_session_user_context_resolves_affiliations(app): + with app.test_request_context("/results"): + from flask import session + + session["authenticated"] = True + session["user_email"] = "user@example.com" + context = get_session_user_context() + + assert context["authenticated"] is True + assert context["email"] == "user@example.com" + assert context["affiliations"] == ["dev"] diff --git a/result_server/utils/node_hours.py b/result_server/utils/node_hours.py index 3552a37..737d605 100644 --- a/result_server/utils/node_hours.py +++ b/result_server/utils/node_hours.py @@ -4,19 +4,16 @@ import os import re from datetime import datetime -from typing import Optional def compute_node_hours(data: dict) -> float: """ - 単一Result JSONからノード時間を算出する。 + Compute node-hours from a Result JSON record. - - cross モード: node_count × run_time / 3600 - - native モード: node_count × (build_time + run_time) / 3600 - - node_count/run_time が欠損・非数値の場合は 0.0 - - native で build_time が欠損・非数値の場合は build_time=0 として算出 - - Returns: float（小数点以下2桁に丸め） + - `cross`: `node_count * run_time / 3600` + - `native`: `node_count * (build_time + run_time) / 3600` + - Missing or invalid `node_count` / `run_time` returns `0.0` + - Missing or invalid `build_time` in native mode falls back to `0.0` """ try: node_count = float(data.get("node_count", None)) @@ -39,16 +36,15 @@ def compute_node_hours(data: dict) -> float: build_time = 0.0 return round(node_count * (build_time + run_time) / 3600, 2) - # cross モード（デフォルト） return round(node_count * run_time / 3600, 2) def extract_timestamp_from_filename(filename: str) -> datetime | None: """ - ファイル名から YYYYMMDD_HHMMSS パターンのタイムスタンプを抽出する。 - 既存の results_loader.py と同じ正規表現パターンを使用。 + Extract a `YYYYMMDD_HHMMSS` timestamp from a result filename. - Returns: datetime or None + This is shared with the results loader so timestamp formatting stays + consistent across portal pages. """ match = re.search(r"\d{8}_\d{6}", filename) if not match: @@ -60,28 +56,19 @@ def extract_timestamp_from_filename(filename: str) -> datetime | None: def get_fiscal_year(dt: datetime) -> int: - """ - 日付から会計年度を返す。 - 1〜3月 → 前年の会計年度、4〜12月 → その年の会計年度 - """ + """Return the Japanese fiscal year for a date.""" if dt.month <= 3: return dt.year - 1 return dt.year def get_fiscal_month_index(dt: datetime) -> int: - """ - 会計年度内の月インデックス（0〜11）を返す。 - 4月=0, 5月=1, ..., 3月=11 - """ + """Return the fiscal-month index in the range `0..11`.""" return (dt.month - 4) % 12 def get_half(dt: datetime) -> str: - """ - 上期（first）か下期（second）かを返す。 - 4〜9月=first, 10〜3月=second - """ + """Return `first` for April-September, otherwise `second`.""" if 4 <= dt.month <= 9: return "first" return "second" @@ -89,40 +76,32 @@ def get_half(dt: datetime) -> str: def _generate_period_labels(fiscal_year: int, period_type: str) -> list[str]: """ - 期間タイプに応じた期間ラベルリストを生成する。 + Generate the display labels for the requested aggregation period. - - monthly: 12個 "YYYY年M月"（4月〜翌3月） - - semi_annual: 2個 "上期（4月〜9月）", "下期（10月〜3月）" - - fiscal_year: 1個 "FY{year}" + - `monthly`: `YYYY-MM` + - `semi_annual`: `H1`, `H2` + - `fiscal_year`: `FY{year}` """ if period_type == "monthly": labels = [] for i in range(12): - month = (4 + i - 1) % 12 + 1 # 4,5,...,12,1,2,3 + month = (4 + i - 1) % 12 + 1 year = fiscal_year if month >= 4 else fiscal_year + 1 - labels.append(f"{year}年{month}月") + labels.append(f"{year}-{month:02d}") return labels - elif period_type == "semi_annual": - return ["上期（4月〜9月）", "下期（10月〜3月）"] - else: # fiscal_year - return [f"FY{fiscal_year}"] + if period_type == "semi_annual": + return ["H1", "H2"] + return [f"FY{fiscal_year}"] def _get_period_key(dt: datetime, period_type: str) -> str | None: - """ - 日付と期間タイプから対応する期間ラベルキーを返す。 - """ + """Map a datetime to the corresponding period label.""" if period_type == "monthly": - return f"{dt.year}年{dt.month}月" - elif period_type == "semi_annual": - half = get_half(dt) - if half == "first": - return "上期（4月〜9月）" - else: - return "下期（10月〜3月）" - else: # fiscal_year - fy = get_fiscal_year(dt) - return f"FY{fy}" + return f"{dt.year}-{dt.month:02d}" + if period_type == "semi_annual": + return "H1" if get_half(dt) == "first" else "H2" + fy = get_fiscal_year(dt) + return f"FY{fy}" def aggregate_node_hours( @@ -131,33 +110,18 @@ def aggregate_node_hours( period_type: str, ) -> dict: """ - 指定ディレクトリの全JSONファイルを読み込み、ノード時間をクロス集計する。 - confidentialフィルタなし（admin専用ページのため全データ対象）。 - - Args: - directory: JSONファイルのあるディレクトリパス - fiscal_year: 対象会計年度 - period_type: "monthly" | "semi_annual" | "fiscal_year" - - Returns: { - "apps": [...], - "systems": [...], - "periods": [...], - "table": {app: {system: {period: float}}}, - "row_totals": {app: {period: float}}, - "col_totals": {system: {period: float}}, - "grand_totals": {period: float}, - "available_fiscal_years": [2024, 2025, ...], - } + Aggregate node-hours from all Result JSON files in a directory. + + Confidential filtering is handled by the admin-only route that calls + this helper, so this function intentionally aggregates every result + present in the directory. """ periods = _generate_period_labels(fiscal_year, period_type) - # 全JSONファイルを走査 apps_set: set[str] = set() systems_set: set[str] = set() all_fiscal_years: set[int] = set() - # table[app][system][period] = float table: dict[str, dict[str, dict[str, float]]] = {} try: @@ -165,10 +129,9 @@ def aggregate_node_hours( except OSError: files = [] - json_files = [f for f in files if f.endswith(".json")] + json_files = [filename for filename in files if filename.endswith(".json")] for json_file in json_files: - # タイムスタンプ抽出 ts = extract_timestamp_from_filename(json_file) if ts is None: continue @@ -176,16 +139,13 @@ def aggregate_node_hours( file_fy = get_fiscal_year(ts) all_fiscal_years.add(file_fy) - # 対象会計年度でなければスキップ if file_fy != fiscal_year: continue - # 期間キーを取得 period_key = _get_period_key(ts, period_type) if period_key not in periods: continue - # JSONファイル読み込み filepath = os.path.join(directory, json_file) try: with open(filepath, "r", encoding="utf-8") as f: @@ -203,18 +163,15 @@ def aggregate_node_hours( apps_set.add(app) systems_set.add(system) - # テーブルに加算 if app not in table: table[app] = {} if system not in table[app]: table[app][system] = {} table[app][system][period_key] = table[app][system].get(period_key, 0.0) + node_hours - # ソート apps = sorted(apps_set) systems = sorted(systems_set) - # 全app×system×periodのセルを初期化（存在しないセルは0.0） for app in apps: if app not in table: table[app] = {} @@ -225,7 +182,6 @@ def aggregate_node_hours( if period not in table[app][system]: table[app][system][period] = 0.0 - # row_totals: 各appの期間別合計（全systemの合計） row_totals: dict[str, dict[str, float]] = {} for app in apps: row_totals[app] = {} @@ -233,7 +189,6 @@ def aggregate_node_hours( total = sum(table[app][system][period] for system in systems) row_totals[app][period] = round(total, 2) - # col_totals: 各systemの期間別合計（全appの合計） col_totals: dict[str, dict[str, float]] = {} for system in systems: col_totals[system] = {} @@ -241,13 +196,11 @@ def aggregate_node_hours( total = sum(table[app][system][period] for app in apps) col_totals[system][period] = round(total, 2) - # grand_totals: 期間別の総合計 grand_totals: dict[str, float] = {} for period in periods: total = sum(row_totals[app][period] for app in apps) grand_totals[period] = round(total, 2) - # セル値も丸める for app in apps: for system in systems: for period in periods: diff --git a/result_server/utils/result_compare_view.py b/result_server/utils/result_compare_view.py index f55a210..8d021df 100644 --- a/result_server/utils/result_compare_view.py +++ b/result_server/utils/result_compare_view.py @@ -1,9 +1,24 @@ +from utils.result_file import check_file_permission +from utils.result_records import load_result_json_batch + + def build_result_compare_context(results): has_vector_metrics = any( (row.get("data") or {}).get("metrics", {}).get("vector") for row in results ) + mixed = False + if results: + first = results[0].get("data", {}) + first_system = first.get("system") + first_code = first.get("code") + mixed = any( + (row.get("data") or {}).get("system") != first_system + or (row.get("data") or {}).get("code") != first_code + for row in results[1:] + ) + headline = "" if results: first = results[0].get("data", {}) @@ -13,4 +28,12 @@ def build_result_compare_context(results): "results": results, "headline": headline, "has_vector_metrics": has_vector_metrics, + "mixed": mixed, } + + +def load_result_compare_context(filenames, directory): + for filename in filenames: + check_file_permission(filename, directory) + results = load_result_json_batch(filenames, directory) + return build_result_compare_context(results) diff --git a/result_server/utils/result_file.py b/result_server/utils/result_file.py index 1d82c0f..a299a9b 100644 --- a/result_server/utils/result_file.py +++ b/result_server/utils/result_file.py @@ -6,7 +6,7 @@ from flask import Response, abort, send_from_directory, session -from utils.user_store import get_user_store +from utils.session_user_context import get_session_user_context def load_result_file(filename: str, save_dir: str): @@ -56,10 +56,9 @@ def check_file_permission(filename: str, dir_path: str) -> None: if not tags: return - authenticated = session.get("authenticated", False) - email = session.get("user_email") - store = get_user_store() - affiliations = store.get_affiliations(email) if email else [] + user_context = get_session_user_context() + authenticated = user_context["authenticated"] + affiliations = user_context["affiliations"] if not authenticated or not (set(tags) & set(affiliations)): abort(403, "You do not have permission to access this file") diff --git a/result_server/utils/session_user_context.py b/result_server/utils/session_user_context.py new file mode 100644 index 0000000..7ab135b --- /dev/null +++ b/result_server/utils/session_user_context.py @@ -0,0 +1,17 @@ +from flask import session + +from utils.user_store import get_user_store + + +def get_session_user_context(): + """Return the current session authentication state and resolved affiliations.""" + authenticated = session.get("authenticated", False) + email = session.get("user_email") + affiliations = [] + if authenticated and email: + affiliations = get_user_store().get_affiliations(email) + return { + "authenticated": authenticated, + "email": email, + "affiliations": affiliations, + } diff --git a/result_server/utils/user_store.py b/result_server/utils/user_store.py index 4cdaf35..0931be8 100644 --- a/result_server/utils/user_store.py +++ b/result_server/utils/user_store.py @@ -1,32 +1,25 @@ -"""Redisベースのユーザーストア - -ユーザーCRUDと招待トークン管理を提供する。 -Redisキー構造: - {prefix}:users → Set型: 登録済みメールアドレスの集合 - {prefix}:user:{email}:totp_secret → String型: Base32エンコードされたTOTP秘密鍵 - {prefix}:user:{email}:affiliations → List型: 所属グループのリスト - {prefix}:invitation:{token} → Hash型: {email, affiliations} (TTL: 24時間) -""" +"""Redis-backed user and invitation storage helpers.""" import secrets from typing import Dict, List, Optional from flask import current_app -INVITATION_TTL = 86400 # 24時間 +INVITATION_TTL = 86400 # 24 hours class UserStore: def __init__(self, redis_conn, key_prefix: str = ""): """ Args: - redis_conn: Redisクライアント接続 - key_prefix: 環境ごとのキープレフィックス（"main:" or "dev:"） + redis_conn: Redis client instance. + key_prefix: Prefix used to separate environments, for example + ``main:`` or ``dev:``. """ self.r = redis_conn self.prefix = key_prefix - # --- キーヘルパー --- + # --- Key helpers --- def _key(self, *parts: str) -> str: return f"{self.prefix}{''.join(parts)}" @@ -43,16 +36,17 @@ def _affiliations_key(self, email: str) -> str: def _invitation_key(self, token: str) -> str: return self._key(f"invitation:{token}") - # --- ユーザー管理 --- + # --- User operations --- def create_user( self, email: str, totp_secret: str, affiliations: List[str] ) -> None: - """ユーザーを登録する。usersセットへの追加、秘密鍵・所属情報の保存。""" + """Create a user with a TOTP secret and affiliations.""" pipe = self.r.pipeline() pipe.sadd(self._users_key(), email) pipe.set(self._secret_key(email), totp_secret) - # 所属情報: 既存リストを削除してから新規追加 + + # Replace the affiliation list instead of appending duplicates. aff_key = self._affiliations_key(email) pipe.delete(aff_key) for aff in affiliations: @@ -60,10 +54,12 @@ def create_user( pipe.execute() def get_user(self, email: str) -> Optional[Dict]: - """ユーザー情報を取得する。 + """ + Return the stored user record. Returns: - {"email": str, "totp_secret": str, "affiliations": list} or None + ``{"email": str, "totp_secret": str, "affiliations": list}`` + or ``None`` when the user does not exist. """ if not self.user_exists(email): return None @@ -76,7 +72,7 @@ def get_user(self, email: str) -> Optional[Dict]: } def delete_user(self, email: str) -> bool: - """ユーザーを削除する。全関連キーを削除。""" + """Delete a user and its related keys.""" if not self.user_exists(email): return False pipe = self.r.pipeline() @@ -87,7 +83,7 @@ def delete_user(self, email: str) -> bool: return True def list_users(self) -> List[Dict]: - """全ユーザーの一覧を返す。""" + """Return every user record in sorted order.""" emails = self.r.smembers(self._users_key()) users = [] for email in sorted(emails): @@ -97,7 +93,7 @@ def list_users(self) -> List[Dict]: return users def update_affiliations(self, email: str, affiliations: List[str]) -> bool: - """ユーザーの所属情報を更新する。""" + """Replace a user's affiliations.""" if not self.user_exists(email): return False aff_key = self._affiliations_key(email) @@ -109,31 +105,32 @@ def update_affiliations(self, email: str, affiliations: List[str]) -> bool: return True def user_exists(self, email: str) -> bool: - """ユーザーが登録済みか確認する。""" + """Return whether a user exists.""" return self.r.sismember(self._users_key(), email) def get_affiliations(self, email: str) -> List[str]: - """ユーザーの所属情報を取得する。""" + """Return the stored affiliations for a user.""" return self.r.lrange(self._affiliations_key(email), 0, -1) def clear_totp_secret(self, email: str) -> bool: - """ユーザーのTOTP秘密鍵を削除する（再登録用）。""" + """Delete the TOTP secret for an existing user.""" if not self.user_exists(email): return False self.r.delete(self._secret_key(email)) return True def has_totp_secret(self, email: str) -> bool: - """ユーザーがTOTP秘密鍵を持っているか確認する。""" + """Return whether the user currently has a TOTP secret.""" return bool(self.r.exists(self._secret_key(email))) - # --- 招待トークン管理 --- + # --- Invitation operations --- def create_invitation(self, email: str, affiliations: List[str]) -> str: - """招待トークンを生成してRedisに保存する。 + """ + Create and persist an invitation token in Redis. Returns: - 招待トークン文字列 + The generated invitation token. """ token = secrets.token_urlsafe(32) key = self._invitation_key(token) @@ -148,10 +145,12 @@ def create_invitation(self, email: str, affiliations: List[str]) -> str: return token def get_invitation(self, token: str) -> Optional[Dict]: - """招待トークンの情報を取得する。 + """ + Return the stored invitation payload. Returns: - {"email": str, "affiliations": list} or None（無効/期限切れ） + ``{"email": str, "affiliations": list}`` or ``None`` when + the invitation does not exist or has expired. """ data = self.r.hgetall(self._invitation_key(token)) if not data: @@ -165,10 +164,10 @@ def get_invitation(self, token: str) -> Optional[Dict]: return {"email": data["email"], "affiliations": affiliations} def delete_invitation(self, token: str) -> None: - """招待トークンを削除する。""" + """Delete an invitation token.""" self.r.delete(self._invitation_key(token)) def get_user_store() -> UserStore: - """現在のアプリコンテキストからUserStoreインスタンスを取得する。""" + """Return the configured application UserStore instance.""" return current_app.config["USER_STORE"] From 9ce03acb962414da184b4f96bbe8acf5aaa7a4bd Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 13:38:53 +0900 Subject: [PATCH 44/89] ci: skip benchmark pipeline for result_server changes [skip-ci] --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f8380a..882f1c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ # # Files that automatically skip CI: # - README.md, ADD_APP.md (documentation) -# - result_server/templates/*.html (web templates) +# - result_server/**/* (portal/server code and templates) # - .kiro/**/* (Kiro configuration) # - .vscode/**/* (VSCode settings) # @@ -96,7 +96,7 @@ generate_matrix: when: always - changes: - "*.md" - - "result_server/templates/*.html" + - "result_server/**/*" - ".kiro/**/*" - ".vscode/**/*" when: never @@ -133,7 +133,7 @@ trigger_child_pipeline: when: always - changes: - "*.md" - - "result_server/templates/*.html" + - "result_server/**/*" - ".kiro/**/*" - ".vscode/**/*" when: never From 18fa2b5c0f3b673e799320e91dd79d704ee85fee Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 13:47:54 +0900 Subject: [PATCH 45/89] refactor: build usage page from view model --- result_server/routes/results.py | 41 ++-------- .../_usage_report_configuration_checks.html | 2 +- .../templates/_usage_report_controls.html | 2 +- .../_usage_report_coverage_section.html | 2 +- .../_usage_report_node_hours_section.html | 2 +- .../_usage_report_quality_section.html | 2 +- result_server/templates/usage_report.html | 2 +- result_server/tests/test_estimated_routes.py | 17 ++-- .../tests/test_portal_list_templates.py | 2 +- result_server/tests/test_usage_report_view.py | 60 ++++++++++++++ result_server/tests/test_usage_route.py | 80 +++++++++++-------- result_server/utils/usage_report_view.py | 36 +++++++++ 12 files changed, 165 insertions(+), 83 deletions(-) create mode 100644 result_server/tests/test_usage_report_view.py create mode 100644 result_server/utils/usage_report_view.py diff --git a/result_server/routes/results.py b/result_server/routes/results.py index 81f732e..345e694 100644 --- a/result_server/routes/results.py +++ b/result_server/routes/results.py @@ -9,19 +9,16 @@ ) from routes.admin import admin_required -from utils.app_support_matrix import load_app_system_support_matrix -from utils.node_hours import aggregate_node_hours, get_fiscal_year +from utils.node_hours import get_fiscal_year from utils.result_compare_view import load_result_compare_context from utils.result_detail_view import build_result_detail_context from utils.result_file import ( load_permitted_result_json, serve_permitted_result_file, ) -from utils.result_quality_rollup import build_result_quality_rollup from utils.result_records import summarize_result_quality from utils.results_loader import DEFAULT_PER_PAGE, get_filter_options, load_results_table from utils.session_user_context import get_session_user_context -from utils.site_diagnostics import build_site_diagnostics from utils.system_info import get_all_systems_info from utils.table_page_utils import ( build_table_page_context, @@ -29,7 +26,7 @@ render_no_store_template, ) from utils.table_query_params import parse_table_query_params -from utils.usage_query_params import parse_usage_query_params, select_usage_periods +from utils.usage_report_view import build_usage_report_context results_bp = Blueprint("results", __name__) @@ -168,36 +165,12 @@ def result_detail(filename): @results_bp.route("/usage", methods=["GET"]) @admin_required def usage_report(): - current_fy = get_fiscal_year(datetime.now()) - params = parse_usage_query_params(request.args, current_fy) - period_type = params["period_type"] - fiscal_year = params["fiscal_year"] - period_filter = params["period_filter"] - - result = aggregate_node_hours(current_app.config["RECEIVED_DIR"], fiscal_year, period_type) - period_filter, filtered_periods = select_usage_periods(result["periods"], period_filter) - - systems_info = get_all_systems_info() - coverage_systems, app_support_rows = load_app_system_support_matrix() - coverage_headers = [ - {"system": system, "name": systems_info.get(system, {}).get("name", system)} - for system in coverage_systems - ] - site_diagnostics = build_site_diagnostics() - result_quality_rollup = build_result_quality_rollup(current_app.config["RECEIVED_DIR"]) - - return render_template( - "usage_report.html", - result=result, - period_type=period_type, - fiscal_year=fiscal_year, - period_filter=period_filter, - filtered_periods=filtered_periods, - coverage_systems=coverage_headers, - app_support_rows=app_support_rows, - site_diagnostics=site_diagnostics, - result_quality_rollup=result_quality_rollup, + usage_context = build_usage_report_context( + current_app.config["RECEIVED_DIR"], + request.args, + get_fiscal_year(datetime.now()), ) + return render_template("usage_report.html", **usage_context) @results_bp.route("/") diff --git a/result_server/templates/_usage_report_configuration_checks.html b/result_server/templates/_usage_report_configuration_checks.html index 726f883..c8a6a97 100644 --- a/result_server/templates/_usage_report_configuration_checks.html +++ b/result_server/templates/_usage_report_configuration_checks.html @@ -1,6 +1,6 @@

Configuration Checks

-

Lightweight checks for registered systems, queue definitions, and application support completeness.

+

Quick checks for registered systems, queues, and support completeness.

Registered Systems

diff --git a/result_server/templates/_usage_report_controls.html b/result_server/templates/_usage_report_controls.html index a9967b7..aa0cceb 100644 --- a/result_server/templates/_usage_report_controls.html +++ b/result_server/templates/_usage_report_controls.html @@ -31,7 +31,7 @@ id="usageSearchInput" class="usage-search-box" type="text" - placeholder="Filter application/system coverage and current-state tables" + placeholder="Filter coverage and current-state tables" oninput="applyUsageSearch()" >
diff --git a/result_server/templates/_usage_report_coverage_section.html b/result_server/templates/_usage_report_coverage_section.html index 283219e..9520d9d 100644 --- a/result_server/templates/_usage_report_coverage_section.html +++ b/result_server/templates/_usage_report_coverage_section.html @@ -1,6 +1,6 @@

Application/System Coverage

-

Current support matrix based on list.csv plus detected build.sh / run.sh system branches.

+

Support matrix from list.csv plus detected build.sh / run.sh branches.

★ enabled and implemented △ enabled in list.csv, script support incomplete diff --git a/result_server/templates/_usage_report_node_hours_section.html b/result_server/templates/_usage_report_node_hours_section.html index 72d6f3d..12158a9 100644 --- a/result_server/templates/_usage_report_node_hours_section.html +++ b/result_server/templates/_usage_report_node_hours_section.html @@ -1,6 +1,6 @@

Node-Hour Usage

-

Aggregated node-hour usage across registered applications, systems, and the selected reporting periods.

+

Aggregated node-hour usage by application, system, and selected period.

{% include "_usage_report_controls.html" %} diff --git a/result_server/templates/_usage_report_quality_section.html b/result_server/templates/_usage_report_quality_section.html index 3ec5567..c40187b 100644 --- a/result_server/templates/_usage_report_quality_section.html +++ b/result_server/templates/_usage_report_quality_section.html @@ -1,6 +1,6 @@

Result Quality Coverage

-

Current-state check based on the latest collected result for each application/system pair, showing source provenance, breakdown output, and estimation-ready fields.

+

Current-state check from the latest result for each application/system pair.

{% if result_quality_rollup.rows %}
diff --git a/result_server/templates/usage_report.html b/result_server/templates/usage_report.html index c5fd273..597bb8b 100644 --- a/result_server/templates/usage_report.html +++ b/result_server/templates/usage_report.html @@ -1,7 +1,7 @@ {% extends "_results_base.html" %} {% block title %}Usage Report{% endblock %} -{% block page_subtitle %}Review node-hour usage, registered coverage, configuration checks, and latest result current-state across applications and systems.{% endblock %} +{% block page_subtitle %}Review node-hour usage, coverage, checks, and latest result current-state across applications and systems.{% endblock %} {% block content %} - - +{% extends "_results_base.html" %} -{% include "_navigation.html" %} +{% block title %}Compare Results{% endblock %} +{% block page_subtitle %}Overlay vector metrics, compare FOM timelines, and sanity-check whether selected runs are from the same system and code.{% endblock %} -← Back to Results +{% block content %} + -

Compare Results

+← Back to Results {% if results %} {% if mixed %} -

+

Warning: Selected results have different system/code combinations. Comparison may not be meaningful.

{% endif %} @@ -132,6 +133,4 @@

FOM Timeline

} {% endif %} - - - +{% endblock %} From b3325a6d41c859a87dda8f8cb9e56f7337421e76 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 17:06:10 +0900 Subject: [PATCH 54/89] refactor: align result detail with shared portal layout --- result_server/routes/results_detail_routes.py | 2 +- result_server/templates/result_detail.html | 144 +++++++++--------- .../tests/test_result_detail_template.py | 4 +- result_server/utils/result_detail_view.py | 4 +- 4 files changed, 72 insertions(+), 82 deletions(-) diff --git a/result_server/routes/results_detail_routes.py b/result_server/routes/results_detail_routes.py index d053fa1..7f42093 100644 --- a/result_server/routes/results_detail_routes.py +++ b/result_server/routes/results_detail_routes.py @@ -29,7 +29,7 @@ def result_detail(filename): not_found_message="Result file not found", ) quality = summarize_result_quality(result) - detail_context = build_result_detail_context(result, quality, filename) + detail_context = build_result_detail_context(result, quality) return render_template("result_detail.html", result=result, quality=quality, **detail_context) @results_bp.route("/") diff --git a/result_server/templates/result_detail.html b/result_server/templates/result_detail.html index 298fa08..5676fc7 100644 --- a/result_server/templates/result_detail.html +++ b/result_server/templates/result_detail.html @@ -1,79 +1,73 @@ - - - - - Result Detail - {{ title_exp }} - {% include "_table_base.html" %} - - - +{% extends "_results_base.html" %} -{% include "_navigation.html" %} +{% block title %}Result Detail{% endblock %} +{% block page_subtitle %}Inspect metrics, profiler summaries, quality signals, and build details for one collected result.{% endblock %} -← Back to Results +{% block content %} + -

Result Detail

+← Back to Results

Meta Information

@@ -281,6 +275,4 @@

Build Information

})(); {% endif %} - - - +{% endblock %} diff --git a/result_server/tests/test_result_detail_template.py b/result_server/tests/test_result_detail_template.py index f051f45..def847d 100644 --- a/result_server/tests/test_result_detail_template.py +++ b/result_server/tests/test_result_detail_template.py @@ -82,10 +82,10 @@ def app(): } -def _render_result_detail(result, quality, filename="test.json"): +def _render_result_detail(result, quality): from flask import render_template - detail_context = build_result_detail_context(result, quality, filename) + detail_context = build_result_detail_context(result, quality) return render_template("result_detail.html", result=result, quality=quality, **detail_context) diff --git a/result_server/utils/result_detail_view.py b/result_server/utils/result_detail_view.py index a576b9f..2377152 100644 --- a/result_server/utils/result_detail_view.py +++ b/result_server/utils/result_detail_view.py @@ -1,15 +1,13 @@ from utils.result_records import format_numeric_value -def build_result_detail_context(result, quality, filename): +def build_result_detail_context(result, quality): profile_data = result.get("profile_data") or {} build_data = result.get("build") or {} vector_metrics = (result.get("metrics") or {}).get("vector") scalar_metrics = (result.get("metrics") or {}).get("scalar") or {} return { - "filename": filename, - "title_exp": result.get("Exp", "Unknown"), "meta_rows": _build_meta_rows(result), "profile_rows": _build_profile_rows(profile_data), "quality_rows": _build_quality_rows(quality), From 0991f13a7fa98d992d3ad43a71adbefd4a275cc3 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 17:15:32 +0900 Subject: [PATCH 55/89] refactor: share portal detail table templates --- .../routes/estimated_detail_routes.py | 4 +- .../templates/_chart_page_styles.html | 7 ++ result_server/templates/_detail_tables.html | 22 ++++++ result_server/templates/_table_base.html | 12 ++-- result_server/templates/estimated_detail.html | 72 +++---------------- result_server/templates/result_compare.html | 5 +- result_server/templates/result_detail.html | 45 ++---------- .../tests/test_estimated_detail_template.py | 7 +- result_server/utils/estimated_detail_view.py | 68 ++++++++++++++++++ 9 files changed, 127 insertions(+), 115 deletions(-) create mode 100644 result_server/templates/_chart_page_styles.html create mode 100644 result_server/templates/_detail_tables.html create mode 100644 result_server/utils/estimated_detail_view.py diff --git a/result_server/routes/estimated_detail_routes.py b/result_server/routes/estimated_detail_routes.py index 6e20f31..3848b01 100644 --- a/result_server/routes/estimated_detail_routes.py +++ b/result_server/routes/estimated_detail_routes.py @@ -4,6 +4,7 @@ load_authenticated_result_json, serve_authenticated_result_file, ) +from utils.estimated_detail_view import build_estimated_detail_context def register_estimated_detail_routes(estimated_bp): @@ -23,4 +24,5 @@ def estimated_detail(filename): message="Authentication required to view estimated data", not_found_message="Estimated result file not found", ) - return render_template("estimated_detail.html", result=result, filename=filename) + detail_context = build_estimated_detail_context(result) + return render_template("estimated_detail.html", result=result, **detail_context) diff --git a/result_server/templates/_chart_page_styles.html b/result_server/templates/_chart_page_styles.html new file mode 100644 index 0000000..9825050 --- /dev/null +++ b/result_server/templates/_chart_page_styles.html @@ -0,0 +1,7 @@ + diff --git a/result_server/templates/_detail_tables.html b/result_server/templates/_detail_tables.html new file mode 100644 index 0000000..f118c47 --- /dev/null +++ b/result_server/templates/_detail_tables.html @@ -0,0 +1,22 @@ +{% macro render_key_value_table(rows, table_class, wrap_class="detail-table-wrap", list_class="packages-list") -%} +
+
+ {% for row in rows %} + + + + {% if row.list is defined %} +
    + {% for item in row.list %} +
  • {{ item }}
    • + {% endfor %} +
+ {% else %} + {{ row.value }} + {% endif %} + + + {% endfor %} +
{{ row.label }}
+
+{%- endmacro %} diff --git a/result_server/templates/_table_base.html b/result_server/templates/_table_base.html index a00ea26..7d9e1b9 100644 --- a/result_server/templates/_table_base.html +++ b/result_server/templates/_table_base.html @@ -1,5 +1,5 @@ - {% set estimate_meta = result.get('estimate_metadata', {}) %} - {% set current = result.get('current_system', {}) %} - {% set future = result.get('future_system', {}) %} - {% set measurement = result.get('measurement', {}) %} - {% set assumptions = result.get('assumptions', {}) %} - {% set confidence = result.get('confidence', {}) %} - {% set current_breakdown = current.get('fom_breakdown', {}) %} - {% set future_breakdown = future.get('fom_breakdown', {}) %} -

Meta Information

-
- - - - - - - - - - - -
Code{{ result.get('code', 'N/A') }}
Exp{{ result.get('exp', 'N/A') }}
Applicability{{ result.get('applicability', {}).get('status', 'N/A') }}
Requested Package{{ estimate_meta.get('requested_estimation_package', 'N/A') }}
Applied Package{{ estimate_meta.get('estimation_package', 'N/A') }}
Method Class{{ estimate_meta.get('method_class', 'N/A') }}
Detail Level{{ estimate_meta.get('detail_level', 'N/A') }}
Estimate UUID{{ estimate_meta.get('estimation_result_uuid', 'N/A') }}
Estimate Timestamp{{ estimate_meta.get('estimation_result_timestamp', 'N/A') }}
Performance Ratio{{ "%.3f"|format(result.get('performance_ratio', 'N/A')|float) if result.get('performance_ratio') not in [None, '', 'N/A', 'null', 'nan'] else result.get('performance_ratio', 'N/A') }}
-
+ {{ render_key_value_table(meta_rows, "detail-table", "table-wrap") }}

Package Resolution

-
- - - - - -
Current Requested{{ estimate_meta.get('current_package', {}).get('requested_estimation_package', 'N/A') }}
Current Applied{{ estimate_meta.get('current_package', {}).get('estimation_package', 'N/A') }}
Future Requested{{ estimate_meta.get('future_package', {}).get('requested_estimation_package', 'N/A') }}
Future Applied{{ estimate_meta.get('future_package', {}).get('estimation_package', 'N/A') }}
-
+ {{ render_key_value_table(package_rows, "detail-table", "table-wrap") }}

Measurement

-
{{ measurement | tojson(indent=2) }}
+
{{ measurement_json | tojson(indent=2) }}

Confidence

-
{{ confidence | tojson(indent=2) }}
+
{{ confidence_json | tojson(indent=2) }}

Current System

-
- - - - - - - - - - - - -
System{{ current.get('system', 'N/A') }}
FOM{{ current.get('fom', 'N/A') }}
Target Nodes{{ current.get('target_nodes', 'N/A') }}
Scaling Method{{ current.get('scaling_method', 'N/A') }}
Benchmark System{{ current.get('benchmark', {}).get('system', 'N/A') }}
Benchmark FOM{{ current.get('benchmark', {}).get('fom', 'N/A') }}
Benchmark Nodes{{ current.get('benchmark', {}).get('nodes', 'N/A') }}
Sections{{ current_breakdown.get('sections', []) | length }}
Overlaps{{ current_breakdown.get('overlaps', []) | length }}
Model{{ current.get('model', {}).get('name', 'N/A') }}
Model Type{{ current.get('model', {}).get('type', 'N/A') }}
-
+ {{ render_key_value_table(current_rows, "detail-table", "table-wrap") }}

Future System

-
- - - - - - - - - - - - -
System{{ future.get('system', 'N/A') }}
FOM{{ future.get('fom', 'N/A') }}
Target Nodes{{ future.get('target_nodes', 'N/A') }}
Scaling Method{{ future.get('scaling_method', 'N/A') }}
Benchmark System{{ future.get('benchmark', {}).get('system', 'N/A') }}
Benchmark FOM{{ future.get('benchmark', {}).get('fom', 'N/A') }}
Benchmark Nodes{{ future.get('benchmark', {}).get('nodes', 'N/A') }}
Sections{{ future_breakdown.get('sections', []) | length }}
Overlaps{{ future_breakdown.get('overlaps', []) | length }}
Model{{ future.get('model', {}).get('name', 'N/A') }}
Model Type{{ future.get('model', {}).get('type', 'N/A') }}
-
+ {{ render_key_value_table(future_rows, "detail-table", "table-wrap") }}

Assumptions

-
{{ assumptions | tojson(indent=2) }}
+
{{ assumptions_json | tojson(indent=2) }}
diff --git a/result_server/templates/result_compare.html b/result_server/templates/result_compare.html index 0189aa7..122de40 100644 --- a/result_server/templates/result_compare.html +++ b/result_server/templates/result_compare.html @@ -4,11 +4,8 @@ {% block page_subtitle %}Overlay vector metrics, compare FOM timelines, and sanity-check whether selected runs are from the same system and code.{% endblock %} {% block content %} +{% include "_chart_page_styles.html" %}
diff --git a/result_server/templates/_usage_report_quality_section.html b/result_server/templates/_usage_report_quality_section.html index c40187b..9190865 100644 --- a/result_server/templates/_usage_report_quality_section.html +++ b/result_server/templates/_usage_report_quality_section.html @@ -39,6 +39,6 @@

Result Quality Coverage

{% else %} -

No collected result-quality data is available yet.

+

No collected result-quality data is available yet.

{% endif %}
diff --git a/result_server/templates/estimated_results.html b/result_server/templates/estimated_results.html index 41d4e39..b5d063e 100644 --- a/result_server/templates/estimated_results.html +++ b/result_server/templates/estimated_results.html @@ -59,7 +59,7 @@ {% include "_pagination.html" %} - + + onerror="document.getElementById('chartFallback').hidden = false; document.getElementById('vectorChart').style.display = 'none';"> - - - -{% include "_navigation.html" %} - -
-
- CX Portal -

Available Systems

-

This page summarizes the systems currently connected to the CX portal for browsing, comparison, and onboarding guidance.

-
+ applySystemFilters(); + } -
-
-
- {{ systems_summary.total_count }} - Connected systems registered in the portal. -
-
- {{ systems_summary.gpu_enabled_count }} - Systems with GPU accelerators listed in system_info.csv. -
-
- {{ systems_summary.cpu_only_count }} - CPU-only systems available for benchmark onboarding and browsing. -
-
+ document.addEventListener("DOMContentLoaded", () => { + applySystemFilters(); + }); + -
- - - - - {{ systems_summary.total_count }} visible +
+
+
+ {{ systems_summary.total_count }} + Connected systems registered in the portal.
- -
- - - - - - - - - - - - - - {% for system_name, info in systems_info.items() %} - - - - - - - - - - {% endfor %} - -
SYSTEMCPU NameCPU/nodeCPU Core CountGPU NameGPU/nodeMemory
{{ info.name }}{{ info.cpu_name }}{{ info.cpu_per_node }}{{ info.cpu_cores }}{{ info.gpu_name }}{{ info.gpu_per_node }}{{ info.memory }}
+
+ {{ systems_summary.gpu_enabled_count }} + Systems with GPU accelerators listed in system_info.csv.
-

- Hardware summaries are sourced from config/system_info.csv. Use this page for quick orientation, then follow the onboarding guides when adding new systems or applications. -

-
- +
+ {{ systems_summary.cpu_only_count }} + CPU-only systems available for benchmark onboarding and browsing.
-
-
+
+ +
+ + + + + {{ systems_summary.total_count }} visible +
- - +
+ + + + + + + + + + + + + + {% for system_name, info in systems_info.items() %} + + + + + + + + + + {% endfor %} + +
SYSTEMCPU NameCPU/nodeCPU Core CountGPU NameGPU/nodeMemory
{{ info.name }}{{ info.cpu_name }}{{ info.cpu_per_node }}{{ info.cpu_cores }}{{ info.gpu_name }}{{ info.gpu_per_node }}{{ info.memory }}
+
+

+ Hardware summaries are sourced from config/system_info.csv. Use this page for quick orientation, then follow the onboarding guides when adding new systems or applications. +

+
+ +
+
+{% endblock %} diff --git a/result_server/utils/estimated_table_rows.py b/result_server/utils/estimated_table_rows.py index e460bc7..ae3a6ca 100644 --- a/result_server/utils/estimated_table_rows.py +++ b/result_server/utils/estimated_table_rows.py @@ -1,5 +1,16 @@ from utils.result_records import format_numeric_value, split_display_timestamp +SCALING_SHORT_NAMES = { + "instrumented-app-sections-dummy": "instr-app-sec", + "scale-mock": "scale-mock", +} + +PACKAGE_SHORT_NAMES = { + "instrumented_app_sections_dummy": "instr_app_sec", + "lightweight_fom_scaling": "weakscaling", + "weakscaling": "weakscaling", +} + def build_estimated_table_row(filename, result_data, fallback_uuid=None, fallback_timestamp=None): current = result_data.get("current_system", {}) @@ -37,7 +48,7 @@ def build_estimated_table_row(filename, result_data, fallback_uuid=None, fallbac "requested_current_estimation_package": requested_current, "requested_future_estimation_package": requested_future, "estimate_uuid": estimate_uuid, - "estimate_uuid_short": estimate_uuid[:8] if estimate_uuid else "", + "estimate_uuid_short": _short_identifier(estimate_uuid), "performance_ratio": result_data.get("performance_ratio", ""), "performance_ratio_display": format_numeric_value(result_data.get("performance_ratio", "")), "json_link": filename, @@ -113,43 +124,47 @@ def _build_estimated_system_columns(group_label, key_prefix): def _format_scaling_short_name(value): - if value == "instrumented-app-sections-dummy": - return "instr-app-sec" - if value == "scale-mock": - return "scale-mock" - return value + return SCALING_SHORT_NAMES.get(value, value) def _format_package_short_name(value): - if value == "instrumented_app_sections_dummy": - return "instr_app_sec" - if value in ("lightweight_fom_scaling", "weakscaling"): - return "weakscaling" - return value + return PACKAGE_SHORT_NAMES.get(value, value) def _build_requested_package_title(requested_package, requested_current, requested_future): - title = requested_package or "" - if requested_current: - title += "\ncurrent-side: " + requested_current - if requested_future: - title += "\nfuture-side: " + requested_future - return title + return _build_multiline_title( + requested_package, + [ + ("current-side", requested_current), + ("future-side", requested_future), + ], + ) def _build_applied_package_title(applied_package, method_class, detail_level, current_package, future_package): - title = applied_package or "" - if method_class: - title += "\nclass: " + method_class - if detail_level: - title += "\ndetail: " + detail_level - if current_package: - title += "\ncurrent-side: " + current_package - if future_package: - title += "\nfuture-side: " + future_package - return title + return _build_multiline_title( + applied_package, + [ + ("class", method_class), + ("detail", detail_level), + ("current-side", current_package), + ("future-side", future_package), + ], + ) def _build_applied_package_meta_line(method_class, detail_level): meta_parts = [part for part in (method_class, detail_level) if part] return " / ".join(meta_parts) + + +def _build_multiline_title(base, labeled_values): + lines = [base] if base else [] + for label, value in labeled_values: + if value: + lines.append(f"{label}: {value}") + return "\n".join(lines) + + +def _short_identifier(value, length=8): + return value[:length] if value else "" From 7ff14d8495869bb7e60d0aa6880f9c820a6e8640 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 18:02:53 +0900 Subject: [PATCH 62/89] refactor: simplify portal home shell --- result_server/templates/home.html | 738 +++++++++----------- result_server/utils/estimated_table_rows.py | 25 +- result_server/utils/result_records.py | 12 + 3 files changed, 344 insertions(+), 431 deletions(-) diff --git a/result_server/templates/home.html b/result_server/templates/home.html index 33c8774..5c0c494 100644 --- a/result_server/templates/home.html +++ b/result_server/templates/home.html @@ -1,432 +1,340 @@ - - - - - CX Portal - - - +{% extends "_results_base.html" %} -{% include "_navigation.html" %} +{% block title %}CX Portal{% endblock %} +{% block page_title %}CX Data Flow, Connected Systems, and Portal Access{% endblock %} +{% block page_subtitle %}This portal is part of the CX Framework, supporting continuous benchmarking, estimation, and feedback for integrated HPC applications.{% endblock %} -
-
-
- CX Portal -

CX Data Flow, Connected Systems, and Portal Access

-

- This portal is part of the overall system implementing the CX Framework, supporting - continuous benchmarking, continuous estimation, and continuous feedback for HPC - applications integrated into this portal. -

-
- Browse Results - View All Systems - {% if session.get('authenticated') %} - Open Estimated Results - {% else %} - Estimated Results (login required) - {% endif %} -
+{% block content %} + + +
+
+

Start Here

+

+ Browse measured and estimated results, inspect connected systems, and jump directly into onboarding guides for applications and sites. +

+
+ Browse Results + View All Systems + {% if session.get('authenticated') %} + Open Estimated Results + {% else %} + Estimated Results (login required) + {% endif %}
+
-
- -
-
-

Main Entry Points

-

- Start here for the main portal pages used to browse measured results, projected outputs, and connected systems. -

-
- - Results - Browse measured benchmark results, compare runs, and inspect result-quality detail pages. - - - {% if session.get('authenticated') %} - Estimated Results - Review system projections, current/future breakdowns, and estimation detail derived from benchmark inputs. - {% else %} - Estimated Results - Review system projections and estimation detail derived from benchmark inputs after signing in. - Login required - {% endif %} - - - Systems - See connected systems, quick hardware summaries, and onboarding-facing system metadata. - +
+ Results + Measured and projected views
-
- -
-

For Application Developers

-

- These guides explain how to add a new benchmark application, enable a new site, and attach estimation support. -

-
- - Add a New Application - Define `build.sh`, `run.sh`, `list.csv`, and result emission for a supported benchmark application. - - - Add a New Site - Register new systems, queue templates, display metadata, and site-specific runner setup. - - - Add Estimation Support - Connect application-side result emission to estimation inputs and estimation workflows. - +
+ Guides + Guides for apps and sites
+ +
- {% if session.get('authenticated') and 'admin' in session.get('user_affiliations', []) %} -
-

Operations Views

-

- These pages help administrators check node-hour usage, application/system coverage, configuration state, and portal access setup. -

-
- - Usage Report - Inspect node-hour usage together with coverage checks, latest result quality state, and source provenance state. - - - User Management - Manage user affiliations, invitation links, and TOTP onboarding for the portal. - -
+
+
+

Main Entry Points

+

+ Start here for the main portal pages used to browse measured results, projected outputs, and connected systems. +

+
+ + Results + Browse measured benchmark results, compare runs, and inspect result-quality detail pages. + + + {% if session.get('authenticated') %} + Estimated Results + Review system projections, current/future breakdowns, and estimation detail derived from benchmark inputs. + {% else %} + Estimated Results + Review system projections and estimation detail derived from benchmark inputs after signing in. + Login required + {% endif %} + + + Systems + See connected systems, quick hardware summaries, and onboarding-facing system metadata. +
- {% endif %}
-
-
-
-

Available Systems

-

Connected systems are sourced from `config/system_info.csv` and shown here for quick orientation.

-
+
+

For Application Developers

+

+ These guides explain how to add a new benchmark application, enable a new site, and attach estimation support. +

+
+ + Add a New Application + Define `build.sh`, `run.sh`, `list.csv`, and result emission for a supported benchmark application. + + + Add a New Site + Register new systems, queue templates, display metadata, and site-specific runner setup. + + + Add Estimation Support + Connect application-side result emission to estimation inputs and estimation workflows. +
-
- {% for system in systems %} -
-
-

{{ system.name }}

- {% if system.gpu_name != '-' %} - GPU-enabled - {% else %} - CPU-only - {% endif %} -
-
-
CPU {{ system.cpu_name }}
-
CPU/node {{ system.cpu_per_node }} | Cores {{ system.cpu_cores }}
-
GPU {{ system.gpu_name }}{% if system.gpu_per_node != '-' %} ({{ system.gpu_per_node }}/node){% endif %}
-
Memory {{ system.memory }}
-
-
- {% endfor %} -
-
- Open the full system list +
+ + {% if session.get('authenticated') and 'admin' in session.get('user_affiliations', []) %} +
+

Operations Views

+

+ These pages help administrators check node-hour usage, application/system coverage, configuration state, and portal access setup. +

+
+ + Usage Report + Inspect node-hour usage together with coverage checks, latest result quality state, and source provenance state. + + + User Management + Manage user affiliations, invitation links, and TOTP onboarding for the portal. +
-
+ {% endif %} +
- - +
+
+
+

Available Systems

+

Connected systems are sourced from `config/system_info.csv` and shown here for quick orientation.

+
+
+
+ {% for system in systems %} +
+
+

{{ system.name }}

+ {% if system.gpu_name != '-' %} + GPU-enabled + {% else %} + CPU-only + {% endif %} +
+
+
CPU {{ system.cpu_name }}
+
CPU/node {{ system.cpu_per_node }} | Cores {{ system.cpu_cores }}
+
GPU {{ system.gpu_name }}{% if system.gpu_per_node != '-' %} ({{ system.gpu_per_node }}/node){% endif %}
+
Memory {{ system.memory }}
+
+
+ {% endfor %} +
+
+ Open the full system list +
+
+{% endblock %} diff --git a/result_server/utils/estimated_table_rows.py b/result_server/utils/estimated_table_rows.py index ae3a6ca..2a19983 100644 --- a/result_server/utils/estimated_table_rows.py +++ b/result_server/utils/estimated_table_rows.py @@ -1,4 +1,9 @@ -from utils.result_records import format_numeric_value, split_display_timestamp +from utils.result_records import ( + build_multiline_title, + format_numeric_value, + short_identifier, + split_display_timestamp, +) SCALING_SHORT_NAMES = { "instrumented-app-sections-dummy": "instr-app-sec", @@ -48,7 +53,7 @@ def build_estimated_table_row(filename, result_data, fallback_uuid=None, fallbac "requested_current_estimation_package": requested_current, "requested_future_estimation_package": requested_future, "estimate_uuid": estimate_uuid, - "estimate_uuid_short": _short_identifier(estimate_uuid), + "estimate_uuid_short": short_identifier(estimate_uuid), "performance_ratio": result_data.get("performance_ratio", ""), "performance_ratio_display": format_numeric_value(result_data.get("performance_ratio", "")), "json_link": filename, @@ -132,7 +137,7 @@ def _format_package_short_name(value): def _build_requested_package_title(requested_package, requested_current, requested_future): - return _build_multiline_title( + return build_multiline_title( requested_package, [ ("current-side", requested_current), @@ -142,7 +147,7 @@ def _build_requested_package_title(requested_package, requested_current, request def _build_applied_package_title(applied_package, method_class, detail_level, current_package, future_package): - return _build_multiline_title( + return build_multiline_title( applied_package, [ ("class", method_class), @@ -156,15 +161,3 @@ def _build_applied_package_title(applied_package, method_class, detail_level, cu def _build_applied_package_meta_line(method_class, detail_level): meta_parts = [part for part in (method_class, detail_level) if part] return " / ".join(meta_parts) - - -def _build_multiline_title(base, labeled_values): - lines = [base] if base else [] - for label, value in labeled_values: - if value: - lines.append(f"{label}: {value}") - return "\n".join(lines) - - -def _short_identifier(value, length=8): - return value[:length] if value else "" diff --git a/result_server/utils/result_records.py b/result_server/utils/result_records.py index 511c2ae..c21dadd 100644 --- a/result_server/utils/result_records.py +++ b/result_server/utils/result_records.py @@ -205,3 +205,15 @@ def split_display_timestamp(value): if " " not in value: return value, "" return value.split(" ", 1) + + +def short_identifier(value, length=8): + return value[:length] if value else "" + + +def build_multiline_title(base, labeled_values): + lines = [base] if base else [] + for label, value in labeled_values: + if value: + lines.append(f"{label}: {value}") + return "\n".join(lines) From 7bf76100d3debec66143af2eddc28853a48e3d59 Mon Sep 17 00:00:00 2001 From: Yoshifumi Nakamura Date: Tue, 14 Apr 2026 18:08:52 +0900 Subject: [PATCH 63/89] fix: simplify portal page headers --- result_server/templates/_navigation.html | 12 ++++++++++++ result_server/templates/_results_base.html | 1 - result_server/templates/_table_base.html | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/result_server/templates/_navigation.html b/result_server/templates/_navigation.html index 1808695..a2e738b 100644 --- a/result_server/templates/_navigation.html +++ b/result_server/templates/_navigation.html @@ -16,6 +16,17 @@ } .nav-left { display: flex; align-items: stretch; gap: 0; } .nav-right { margin-left: auto; display: flex; align-items: center; gap: 4px; } +.nav-brand { + display: inline-flex; + align-items: center; + padding: 10px 14px 10px 0; + margin-right: 8px; + color: #0f172a; + text-decoration: none; + font-weight: 700; + letter-spacing: -0.01em; + white-space: nowrap; +} .nav-link { text-decoration: none; color: #0f766e; @@ -88,6 +99,7 @@